sonic: Pinning to more than one core causes significant latency

I am trying to achieve “It allows latency-sensitive programs to run in a hot-loop pinned to a thread on an isolated core in order to achieve low latency and jitter.”.

Usually I do this by calling LockOSThread and SetSchedulerAffinity (single CPU), but for some reason this introduces significantly latency spikes (>10ms) compared to running it without pinning it. go trace confirms that the Runner is locked to a single thread. Message handlers are not doing a lot more than setting on a poller diode.

Nothing changes when I:

  • don’t call SetSchedulerAffinity
  • remove the core from isolcpus
  • remove the core from nohz_full
  • remove the core from rcu_nocbs

What is the recommended approach to do this?

type Runner struct {
	commandsDiode diodes.Diode

	pollingEnabled bool

	cpuAffinityManager *cpuaffinity.Manager

	l *logger.Logger
}

func NewRunner(
	pollingEnabled bool,
	cpuAffinityManager *cpuaffinity.Manager,
	l *logger.Logger,
) (*Runner, error) {
	zl := l.With().Str("topic", "SonicRunner").Logger()
	l = &logger.Logger{Logger: &zl}

	commandsDiode := diodes.NewPoller(diodes.NewManyToOne(2048, diodes.AlertFunc(func(missed int) {
		l.Error().Msgf("dropped %d messages", missed)
	})))

	return &Runner{
		commandsDiode,
		pollingEnabled,
		cpuAffinityManager,
		l,
	}, nil
}

func (r *Runner) Run() {
	ioc := sonic.MustIO()
	defer ioc.Close()

	if r.pollingEnabled {
		runtime.LockOSThread()
		defer runtime.UnlockOSThread()

		if err := r.cpuAffinityManager.AcquireCPU(); err != nil {
			switch {
			case errors.Is(err, cpuaffinity.ErrOSNotSupported):
				r.l.Info().Err(err).Msgf("could not acquire CPU affinity")
			default:
				r.l.Fatal().Err(err).Msgf("could not acquire CPU affinity")
			}
		}

		defer func() {
			if err := r.cpuAffinityManager.ReleaseCPU(); err != nil {
				r.l.Error().Err(err).Msgf("could not release CPU affinity")
			}
		}()
	}

	i := 0

	for {
		i += 1

		if i%1000 == 0 {
			r.processCommands(ioc)

			i = 0
		}

		if _, err := ioc.PollOne(); err != nil && !errors.Is(err, sonicerrors.ErrTimeout) {
			r.l.Error().Err(err).Msg("could not poll")
		}
	}
}

func (r *Runner) CreateStream(url string, resultCh chan interface{}) {
	r.commandsDiode.Set(diodes.GenericDataType(newCreateStreamCommand(url, resultCh)))
}

func (r *Runner) ReadStream(stream *websocket.WebsocketStream, onRead websocket.AsyncMessageHandler, b []byte) {
	r.commandsDiode.Set(diodes.GenericDataType(newReadStreamCommand(stream, onRead, b)))
}

func (r *Runner) CloseStream(s *websocket.WebsocketStream, resultCh chan interface{}) {
	r.commandsDiode.Set(diodes.GenericDataType(newCloseStreamCommand(s, resultCh)))
}

func (r *Runner) processCommands(ioc *sonic.IO) {
	// read channel may be notified once for multiple messages if we can't keep up
	for {
		p, ok := r.commandsDiode.TryNext()
		if !ok {
			break
		}

		c := (*command)(p)

		switch c.commandType {
		case commandTypeCreateStream:
			r.processCreateStreamCommand(ioc, c)
		case commandTypeReadStream:
			r.processReadStreamCommand(c)
		case commandTypeCloseStream:
			r.processCloseStreamCommand(c)
		default:
			panic("unhandled default case")
		}
	}
}

func (r *Runner) processCreateStreamCommand(ioc *sonic.IO, c *command) {
	stream, err := websocket.NewWebsocketStream(ioc, &tls.Config{}, websocket.RoleClient)
	if err != nil {
		c.resultCh <- fmt.Errorf("could not create websocket stream: %w", err)
		return
	}

	stream.AsyncHandshake(c.url, func(err error) {
		if err != nil {
			c.resultCh <- fmt.Errorf("could not handshake stream: %w", err)
			return
		}

		c.resultCh <- stream
	})
}

func (r *Runner) processReadStreamCommand(c *command) {
	c.stream.AsyncNextMessage(c.b, c.onRead)
}

func (r *Runner) processCloseStreamCommand(c *command) {
	c.stream.AsyncClose(1000, "", func(err error) {
		if err != nil {
			c.resultCh <- fmt.Errorf("could not close stream: %w", err)
			return
		}

		c.resultCh <- nil
	})
}

About this issue

Most upvoted comments

was swamped with work this beginning of the year, but i started testing it today. No conclusion yet, but will follow up with some notes in the coming week

@sergiu128 did you manage to make any progress? Thanks!

Sorry had not time, quite crazy end of the year. Didn’t forget tho, will come back through the holidays on this when I have some time. Thanks!

Would it be possible to implement a quick-and-dirty POC to check whether it would resolve the issue?

Really appreciate your help. See sample of strace output:

epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [{events=EPOLLIN, data={u32=9573552, u64=824643294384}}], 128, 0) = 1
epoll_ctl(3, EPOLL_CTL_DEL, 8, NULL)    = 0
read(8, "\27\3\3\0\252\0\0\0\0\0\0E\251 (^n\302hr\327D\10#\204N7\205Y%<\302"..., 27264) = 175
epoll_ctl(3, EPOLL_CTL_ADD, 8, {events=EPOLLIN, data={u32=9573552, u64=824643294384}}) = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0
epoll_wait(3, [], 128, 0)               = 0

Sysctl

[root@ip-10-2-35-114 ~]# sysctl -a
abi.vsyscall32 = 1
crypto.fips_enabled = 0
crypto.fips_name = Amazon Linux 2023 Kernel Cryptographic API
crypto.fips_version = 6.1.112-122.189.amzn2023.x86_64
debug.exception-trace = 1
debug.kprobes-optimization = 1
dev.hpet.max-user-freq = 64
dev.raid.speed_limit_max = 200000
dev.raid.speed_limit_min = 1000
dev.tty.ldisc_autoload = 0
fs.aio-max-nr = 65536
fs.aio-nr = 0
fs.binfmt_misc.status = enabled
fs.dentry-state = 35161	16111	45	0	1374	0
fs.dir-notify-enable = 1
fs.epoll.max_user_watches = 1756360
fs.fanotify.max_queued_events = 16384
fs.fanotify.max_user_groups = 128
fs.fanotify.max_user_marks = 64735
fs.file-max = 9223372036854775807
fs.file-nr = 1408	0	9223372036854775807
fs.inode-nr = 33751	394
fs.inode-state = 33751	394	0	0	0	0	0
fs.inotify.max_queued_events = 16384
fs.inotify.max_user_instances = 128
fs.inotify.max_user_watches = 60835
fs.lease-break-time = 45
fs.leases-enable = 1
fs.mount-max = 100000
fs.mqueue.msg_default = 10
fs.mqueue.msg_max = 10
fs.mqueue.msgsize_default = 8192
fs.mqueue.msgsize_max = 8192
fs.mqueue.queues_max = 256
fs.nr_open = 1073741816
fs.overflowgid = 65534
fs.overflowuid = 65534
fs.pipe-max-size = 1048576
fs.pipe-user-pages-hard = 0
fs.pipe-user-pages-soft = 16384
fs.protected_fifos = 1
fs.protected_hardlinks = 1
fs.protected_regular = 1
fs.protected_symlinks = 1
fs.quota.allocated_dquots = 0
fs.quota.cache_hits = 0
fs.quota.drops = 0
fs.quota.free_dquots = 0
fs.quota.lookups = 0
fs.quota.reads = 0
fs.quota.syncs = 0
fs.quota.writes = 0
fs.suid_dumpable = 0
fs.xfs.error_level = 3
fs.xfs.filestream_centisecs = 3000
fs.xfs.inherit_noatime = 1
fs.xfs.inherit_nodefrag = 1
fs.xfs.inherit_nodump = 1
fs.xfs.inherit_nosymlinks = 0
fs.xfs.inherit_sync = 1
fs.xfs.irix_sgid_inherit = 0
fs.xfs.irix_symlink_mode = 0
fs.xfs.panic_mask = 0
fs.xfs.rotorstep = 1
fs.xfs.speculative_cow_prealloc_lifetime = 300
fs.xfs.speculative_prealloc_lifetime = 300
fs.xfs.stats_clear = 0
fs.xfs.xfssyncd_centisecs = 3000
kernel.acct = 4	2	30
kernel.acpi_video_flags = 0
kernel.arch = x86_64
kernel.auto_msgmni = 0
kernel.bootloader_type = 6
kernel.bootloader_version = 38
kernel.bpf_stats_enabled = 0
kernel.cad_pid = 1
kernel.cap_last_cap = 40
kernel.core_pattern = |/usr/lib/systemd/systemd-coredump %P %u %g %s %t %c %h
kernel.core_pipe_limit = 16
kernel.core_uses_pid = 1
kernel.ctrl-alt-del = 0
kernel.dmesg_restrict = 1
kernel.domainname = (none)
kernel.firmware_config.force_sysfs_fallback = 0
kernel.firmware_config.ignore_sysfs_fallback = 0
kernel.ftrace_dump_on_oops = 0
kernel.ftrace_enabled = 1
kernel.hostname = 
kernel.hotplug = /sbin/hotplug
kernel.hung_task_all_cpu_backtrace = 0
kernel.hung_task_check_count = 4194304
kernel.hung_task_check_interval_secs = 0
kernel.hung_task_panic = 0
kernel.hung_task_timeout_secs = 120
kernel.hung_task_warnings = 10
kernel.io_delay_type = 0
kernel.kexec_load_disabled = 0
kernel.keys.gc_delay = 300
kernel.keys.maxbytes = 20000
kernel.keys.maxkeys = 200
kernel.keys.persistent_keyring_expiry = 259200
kernel.keys.root_maxbytes = 25000000
kernel.keys.root_maxkeys = 1000000
kernel.kptr_restrict = 0
kernel.latencytop = 0
kernel.max_lock_depth = 1024
kernel.max_rcu_stall_to_panic = 0
kernel.modprobe = /sbin/modprobe
kernel.modules_disabled = 0
kernel.msg_next_id = -1
kernel.msgmax = 8192
kernel.msgmnb = 16384
kernel.msgmni = 32000
kernel.ngroups_max = 65536
kernel.nmi_watchdog = 0
kernel.ns_last_pid = 3453
kernel.numa_balancing = 0
kernel.numa_balancing_promote_rate_limit_MBps = 65536
kernel.oops_all_cpu_backtrace = 0
kernel.oops_limit = 10000
kernel.osrelease = 6.1.112-122.189.amzn2023.x86_64
kernel.ostype = Linux
kernel.overflowgid = 65534
kernel.overflowuid = 65534
kernel.panic = 5
kernel.panic_on_io_nmi = 0
kernel.panic_on_oops = 1
kernel.panic_on_rcu_stall = 0
kernel.panic_on_unrecovered_nmi = 0
kernel.panic_on_warn = 0
kernel.panic_print = 0
kernel.perf_cpu_time_max_percent = 25
kernel.perf_event_max_contexts_per_stack = 8
kernel.perf_event_max_sample_rate = 100000
kernel.perf_event_max_stack = 127
kernel.perf_event_mlock_kb = 516
kernel.perf_event_paranoid = 2
kernel.pid_max = 4194304
kernel.poweroff_cmd = /sbin/poweroff
kernel.print-fatal-signals = 0
kernel.printk = 8	4	1	7
kernel.printk_delay = 0
kernel.printk_devkmsg = on
kernel.printk_ratelimit = 5
kernel.printk_ratelimit_burst = 10
kernel.pty.max = 4096
kernel.pty.nr = 4
kernel.pty.reserve = 1024
kernel.random.boot_id = cf72862a-da78-4328-a3ff-e6c2d090e30d
kernel.random.entropy_avail = 256
kernel.random.poolsize = 256
kernel.random.urandom_min_reseed_secs = 60
kernel.random.uuid = f8a85fba-6ac4-48ca-8e15-b25c87152159
kernel.random.write_wakeup_threshold = 256
kernel.randomize_va_space = 2
kernel.real-root-dev = 0
kernel.sched_autogroup_enabled = 1
kernel.sched_cfs_bandwidth_slice_us = 5000
kernel.sched_child_runs_first = 0
kernel.sched_deadline_period_max_us = 4194304
kernel.sched_deadline_period_min_us = 100
kernel.sched_rr_timeslice_ms = 100
kernel.sched_rt_period_us = 1000000
kernel.sched_rt_runtime_us = 950000
kernel.sched_schedstats = 0
kernel.seccomp.actions_avail = kill_process kill_thread trap errno user_notif trace log allow
kernel.seccomp.actions_logged = kill_process kill_thread trap errno user_notif trace log
kernel.sem = 32000	1024000000	500	32000
kernel.sem_next_id = -1
kernel.shm_next_id = -1
kernel.shm_rmid_forced = 0
kernel.shmall = 18446744073692774399
kernel.shmmax = 18446744073692774399
kernel.shmmni = 4096
kernel.soft_watchdog = 1
kernel.softlockup_all_cpu_backtrace = 0
kernel.softlockup_panic = 0
kernel.split_lock_mitigate = 1
kernel.stack_tracer_enabled = 0
kernel.sysctl_writes_strict = 1
kernel.sysrq = 16
kernel.tainted = 0
kernel.task_delayacct = 0
kernel.threads-max = 61640
kernel.timer_migration = 1
kernel.traceoff_on_warning = 0
kernel.tracepoint_printk = 0
kernel.unknown_nmi_panic = 0
kernel.unprivileged_bpf_disabled = 1
kernel.usermodehelper.bset = 4294967295	511
kernel.usermodehelper.inheritable = 4294967295	511
kernel.version = #1 SMP PREEMPT_DYNAMIC Tue Oct  8 17:02:11 UTC 2024
kernel.warn_limit = 0
kernel.watchdog = 1
kernel.watchdog_cpumask = 0-3
kernel.watchdog_thresh = 10
kernel.yama.ptrace_scope = 0
net.core.bpf_jit_enable = 1
net.core.bpf_jit_harden = 0
net.core.bpf_jit_kallsyms = 1
net.core.bpf_jit_limit = 528482304
net.core.busy_poll = 0
net.core.busy_read = 0
net.core.default_qdisc = fq_codel
net.core.dev_weight = 64
net.core.dev_weight_rx_bias = 1
net.core.dev_weight_tx_bias = 1
net.core.devconf_inherit_init_net = 0
net.core.fb_tunnels_only_for_init_net = 0
net.core.flow_limit_cpu_bitmap = 0
net.core.flow_limit_table_len = 4096
net.core.gro_normal_batch = 8
net.core.high_order_alloc_disable = 0
net.core.max_skb_frags = 17
net.core.mem_pcpu_rsv = 256
net.core.message_burst = 10
net.core.message_cost = 5
net.core.netdev_budget = 300
net.core.netdev_budget_usecs = 20000
net.core.netdev_max_backlog = 1000
net.core.netdev_rss_key = 9c:7b:5f:c9:42:b0:64:07:1c:f3:82:3d:c8:72:95:e1:a4:16:1c:dd:ed:14:5f:c5:b6:d1:cc:4e:bf:90:b6:0e:b4:96:e7:37:32:c8:7c:55:16:18:fa:c2:9b:2c:c1:c8:46:1f:51:a1
net.core.netdev_tstamp_prequeue = 1
net.core.netdev_unregister_timeout_secs = 10
net.core.optmem_max = 81920
net.core.rmem_default = 212992
net.core.rmem_max = 212992
net.core.rps_sock_flow_entries = 0
net.core.skb_defer_max = 64
net.core.somaxconn = 4096
net.core.tstamp_allow_data = 1
net.core.txrehash = 1
net.core.warnings = 0
net.core.wmem_default = 212992
net.core.wmem_max = 212992
net.core.xfrm_acq_expires = 30
net.core.xfrm_aevent_etime = 10
net.core.xfrm_aevent_rseqth = 2
net.core.xfrm_larval_drop = 1
net.ipv4.cipso_cache_bucket_size = 10
net.ipv4.cipso_cache_enable = 1
net.ipv4.cipso_rbm_optfmt = 0
net.ipv4.cipso_rbm_strictvalid = 1
net.ipv4.conf.all.accept_local = 0
net.ipv4.conf.all.accept_redirects = 1
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.arp_accept = 0
net.ipv4.conf.all.arp_announce = 0
net.ipv4.conf.all.arp_evict_nocarrier = 1
net.ipv4.conf.all.arp_filter = 0
net.ipv4.conf.all.arp_ignore = 0
net.ipv4.conf.all.arp_notify = 0
net.ipv4.conf.all.bc_forwarding = 0
net.ipv4.conf.all.bootp_relay = 0
net.ipv4.conf.all.disable_policy = 0
net.ipv4.conf.all.disable_xfrm = 0
net.ipv4.conf.all.drop_gratuitous_arp = 0
net.ipv4.conf.all.drop_unicast_in_l2_multicast = 0
net.ipv4.conf.all.force_igmp_version = 0
net.ipv4.conf.all.forwarding = 0
net.ipv4.conf.all.igmpv2_unsolicited_report_interval = 10000
net.ipv4.conf.all.igmpv3_unsolicited_report_interval = 1000
net.ipv4.conf.all.ignore_routes_with_linkdown = 0
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.all.mc_forwarding = 0
net.ipv4.conf.all.medium_id = 0
net.ipv4.conf.all.promote_secondaries = 0
net.ipv4.conf.all.proxy_arp = 0
net.ipv4.conf.all.proxy_arp_pvlan = 0
net.ipv4.conf.all.route_localnet = 0
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.all.secure_redirects = 1
net.ipv4.conf.all.send_redirects = 1
net.ipv4.conf.all.shared_media = 1
net.ipv4.conf.all.src_valid_mark = 0
net.ipv4.conf.all.tag = 0
net.ipv4.conf.default.accept_local = 0
net.ipv4.conf.default.accept_redirects = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.default.arp_accept = 0
net.ipv4.conf.default.arp_announce = 0
net.ipv4.conf.default.arp_evict_nocarrier = 1
net.ipv4.conf.default.arp_filter = 0
net.ipv4.conf.default.arp_ignore = 0
net.ipv4.conf.default.arp_notify = 0
net.ipv4.conf.default.bc_forwarding = 0
net.ipv4.conf.default.bootp_relay = 0
net.ipv4.conf.default.disable_policy = 0
net.ipv4.conf.default.disable_xfrm = 0
net.ipv4.conf.default.drop_gratuitous_arp = 0
net.ipv4.conf.default.drop_unicast_in_l2_multicast = 0
net.ipv4.conf.default.force_igmp_version = 0
net.ipv4.conf.default.forwarding = 0
net.ipv4.conf.default.igmpv2_unsolicited_report_interval = 10000
net.ipv4.conf.default.igmpv3_unsolicited_report_interval = 1000
net.ipv4.conf.default.ignore_routes_with_linkdown = 0
net.ipv4.conf.default.log_martians = 0
net.ipv4.conf.default.mc_forwarding = 0
net.ipv4.conf.default.medium_id = 0
net.ipv4.conf.default.promote_secondaries = 1
net.ipv4.conf.default.proxy_arp = 0
net.ipv4.conf.default.proxy_arp_pvlan = 0
net.ipv4.conf.default.route_localnet = 0
net.ipv4.conf.default.rp_filter = 2
net.ipv4.conf.default.secure_redirects = 1
net.ipv4.conf.default.send_redirects = 1
net.ipv4.conf.default.shared_media = 1
net.ipv4.conf.default.src_valid_mark = 0
net.ipv4.conf.default.tag = 0
net.ipv4.conf.enp39s0.accept_local = 0
net.ipv4.conf.enp39s0.accept_redirects = 1
net.ipv4.conf.enp39s0.accept_source_route = 0
net.ipv4.conf.enp39s0.arp_accept = 0
net.ipv4.conf.enp39s0.arp_announce = 0
net.ipv4.conf.enp39s0.arp_evict_nocarrier = 1
net.ipv4.conf.enp39s0.arp_filter = 0
net.ipv4.conf.enp39s0.arp_ignore = 0
net.ipv4.conf.enp39s0.arp_notify = 0
net.ipv4.conf.enp39s0.bc_forwarding = 0
net.ipv4.conf.enp39s0.bootp_relay = 0
net.ipv4.conf.enp39s0.disable_policy = 0
net.ipv4.conf.enp39s0.disable_xfrm = 0
net.ipv4.conf.enp39s0.drop_gratuitous_arp = 0
net.ipv4.conf.enp39s0.drop_unicast_in_l2_multicast = 0
net.ipv4.conf.enp39s0.force_igmp_version = 0
net.ipv4.conf.enp39s0.forwarding = 0
net.ipv4.conf.enp39s0.igmpv2_unsolicited_report_interval = 10000
net.ipv4.conf.enp39s0.igmpv3_unsolicited_report_interval = 1000
net.ipv4.conf.enp39s0.ignore_routes_with_linkdown = 0
net.ipv4.conf.enp39s0.log_martians = 0
net.ipv4.conf.enp39s0.mc_forwarding = 0
net.ipv4.conf.enp39s0.medium_id = 0
net.ipv4.conf.enp39s0.promote_secondaries = 1
net.ipv4.conf.enp39s0.proxy_arp = 0
net.ipv4.conf.enp39s0.proxy_arp_pvlan = 0
net.ipv4.conf.enp39s0.route_localnet = 0
net.ipv4.conf.enp39s0.rp_filter = 2
net.ipv4.conf.enp39s0.secure_redirects = 1
net.ipv4.conf.enp39s0.send_redirects = 1
net.ipv4.conf.enp39s0.shared_media = 1
net.ipv4.conf.enp39s0.src_valid_mark = 0
net.ipv4.conf.enp39s0.tag = 0
net.ipv4.conf.lo.accept_local = 0
net.ipv4.conf.lo.accept_redirects = 1
net.ipv4.conf.lo.accept_source_route = 0
net.ipv4.conf.lo.arp_accept = 0
net.ipv4.conf.lo.arp_announce = 0
net.ipv4.conf.lo.arp_evict_nocarrier = 1
net.ipv4.conf.lo.arp_filter = 0
net.ipv4.conf.lo.arp_ignore = 0
net.ipv4.conf.lo.arp_notify = 0
net.ipv4.conf.lo.bc_forwarding = 0
net.ipv4.conf.lo.bootp_relay = 0
net.ipv4.conf.lo.disable_policy = 1
net.ipv4.conf.lo.disable_xfrm = 1
net.ipv4.conf.lo.drop_gratuitous_arp = 0
net.ipv4.conf.lo.drop_unicast_in_l2_multicast = 0
net.ipv4.conf.lo.force_igmp_version = 0
net.ipv4.conf.lo.forwarding = 0
net.ipv4.conf.lo.igmpv2_unsolicited_report_interval = 10000
net.ipv4.conf.lo.igmpv3_unsolicited_report_interval = 1000
net.ipv4.conf.lo.ignore_routes_with_linkdown = 0
net.ipv4.conf.lo.log_martians = 0
net.ipv4.conf.lo.mc_forwarding = 0
net.ipv4.conf.lo.medium_id = 0
net.ipv4.conf.lo.promote_secondaries = 1
net.ipv4.conf.lo.proxy_arp = 0
net.ipv4.conf.lo.proxy_arp_pvlan = 0
net.ipv4.conf.lo.route_localnet = 0
net.ipv4.conf.lo.rp_filter = 2
net.ipv4.conf.lo.secure_redirects = 1
net.ipv4.conf.lo.send_redirects = 1
net.ipv4.conf.lo.shared_media = 1
net.ipv4.conf.lo.src_valid_mark = 0
net.ipv4.conf.lo.tag = 0
net.ipv4.fib_multipath_hash_fields = 7
net.ipv4.fib_multipath_hash_policy = 0
net.ipv4.fib_multipath_use_neigh = 0
net.ipv4.fib_notify_on_flag_change = 0
net.ipv4.fib_sync_mem = 524288
net.ipv4.fwmark_reflect = 0
net.ipv4.icmp_echo_enable_probe = 0
net.ipv4.icmp_echo_ignore_all = 0
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_errors_use_inbound_ifaddr = 0
net.ipv4.icmp_ignore_bogus_error_responses = 1
net.ipv4.icmp_msgs_burst = 50
net.ipv4.icmp_msgs_per_sec = 1000
net.ipv4.icmp_ratelimit = 1000
net.ipv4.icmp_ratemask = 6168
net.ipv4.igmp_link_local_mcast_reports = 1
net.ipv4.igmp_max_memberships = 20
net.ipv4.igmp_max_msf = 10
net.ipv4.igmp_qrv = 2
net.ipv4.inet_peer_maxttl = 600
net.ipv4.inet_peer_minttl = 120
net.ipv4.inet_peer_threshold = 65664
net.ipv4.ip_autobind_reuse = 0
net.ipv4.ip_default_ttl = 127
net.ipv4.ip_dynaddr = 0
net.ipv4.ip_early_demux = 1
net.ipv4.ip_forward = 0
net.ipv4.ip_forward_update_priority = 1
net.ipv4.ip_forward_use_pmtu = 0
net.ipv4.ip_local_port_range = 32768	60999
net.ipv4.ip_local_reserved_ports = 
net.ipv4.ip_no_pmtu_disc = 0
net.ipv4.ip_nonlocal_bind = 0
net.ipv4.ip_unprivileged_port_start = 1024
net.ipv4.ipfrag_high_thresh = 4194304
net.ipv4.ipfrag_low_thresh = 3145728
net.ipv4.ipfrag_max_dist = 64
net.ipv4.ipfrag_secret_interval = 0
net.ipv4.ipfrag_time = 30
net.ipv4.neigh.default.anycast_delay = 100
net.ipv4.neigh.default.app_solicit = 0
net.ipv4.neigh.default.base_reachable_time_ms = 30000
net.ipv4.neigh.default.delay_first_probe_time = 5
net.ipv4.neigh.default.gc_interval = 30
net.ipv4.neigh.default.gc_stale_time = 60
net.ipv4.neigh.default.gc_thresh1 = 0
net.ipv4.neigh.default.gc_thresh2 = 15360
net.ipv4.neigh.default.gc_thresh3 = 16384
net.ipv4.neigh.default.interval_probe_time_ms = 5000
net.ipv4.neigh.default.locktime = 100
net.ipv4.neigh.default.mcast_resolicit = 0
net.ipv4.neigh.default.mcast_solicit = 3
net.ipv4.neigh.default.proxy_delay = 80
net.ipv4.neigh.default.proxy_qlen = 64
net.ipv4.neigh.default.retrans_time_ms = 1000
net.ipv4.neigh.default.ucast_solicit = 3
net.ipv4.neigh.default.unres_qlen = 101
net.ipv4.neigh.default.unres_qlen_bytes = 212992
net.ipv4.neigh.enp39s0.anycast_delay = 100
net.ipv4.neigh.enp39s0.app_solicit = 0
net.ipv4.neigh.enp39s0.base_reachable_time_ms = 30000
net.ipv4.neigh.enp39s0.delay_first_probe_time = 5
net.ipv4.neigh.enp39s0.gc_stale_time = 60
net.ipv4.neigh.enp39s0.interval_probe_time_ms = 5000
net.ipv4.neigh.enp39s0.locktime = 100
net.ipv4.neigh.enp39s0.mcast_resolicit = 0
net.ipv4.neigh.enp39s0.mcast_solicit = 3
net.ipv4.neigh.enp39s0.proxy_delay = 80
net.ipv4.neigh.enp39s0.proxy_qlen = 64
net.ipv4.neigh.enp39s0.retrans_time_ms = 1000
net.ipv4.neigh.enp39s0.ucast_solicit = 3
net.ipv4.neigh.enp39s0.unres_qlen = 101
net.ipv4.neigh.enp39s0.unres_qlen_bytes = 212992
net.ipv4.neigh.lo.anycast_delay = 100
net.ipv4.neigh.lo.app_solicit = 0
net.ipv4.neigh.lo.base_reachable_time_ms = 30000
net.ipv4.neigh.lo.delay_first_probe_time = 5
net.ipv4.neigh.lo.gc_stale_time = 60
net.ipv4.neigh.lo.interval_probe_time_ms = 5000
net.ipv4.neigh.lo.locktime = 100
net.ipv4.neigh.lo.mcast_resolicit = 0
net.ipv4.neigh.lo.mcast_solicit = 3
net.ipv4.neigh.lo.proxy_delay = 80
net.ipv4.neigh.lo.proxy_qlen = 64
net.ipv4.neigh.lo.retrans_time_ms = 1000
net.ipv4.neigh.lo.ucast_solicit = 3
net.ipv4.neigh.lo.unres_qlen = 101
net.ipv4.neigh.lo.unres_qlen_bytes = 212992
net.ipv4.nexthop_compat_mode = 1
net.ipv4.ping_group_range = 0	2147483647
net.ipv4.raw_l3mdev_accept = 1
net.ipv4.route.error_burst = 500
net.ipv4.route.error_cost = 100
net.ipv4.route.gc_elasticity = 8
net.ipv4.route.gc_interval = 60
net.ipv4.route.gc_min_interval = 0
net.ipv4.route.gc_min_interval_ms = 500
net.ipv4.route.gc_thresh = -1
net.ipv4.route.gc_timeout = 300
net.ipv4.route.max_size = 2147483647
net.ipv4.route.min_adv_mss = 256
net.ipv4.route.min_pmtu = 552
net.ipv4.route.mtu_expires = 600
net.ipv4.route.redirect_load = 2
net.ipv4.route.redirect_number = 9
net.ipv4.route.redirect_silence = 2048
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.tcp_adv_win_scale = 1
net.ipv4.tcp_allowed_congestion_control = reno cubic
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_autocorking = 1
net.ipv4.tcp_available_congestion_control = reno cubic
net.ipv4.tcp_available_ulp = mptcp
net.ipv4.tcp_base_mss = 1024
net.ipv4.tcp_challenge_ack_limit = 2147483647
net.ipv4.tcp_child_ehash_entries = 0
net.ipv4.tcp_comp_sack_delay_ns = 1000000
net.ipv4.tcp_comp_sack_nr = 44
net.ipv4.tcp_comp_sack_slack_ns = 100000
net.ipv4.tcp_congestion_control = cubic
net.ipv4.tcp_dsack = 1
net.ipv4.tcp_early_demux = 1
net.ipv4.tcp_early_retrans = 3
net.ipv4.tcp_ecn = 2
net.ipv4.tcp_ecn_fallback = 1
net.ipv4.tcp_ehash_entries = 65536
net.ipv4.tcp_fack = 0
net.ipv4.tcp_fastopen = 1
net.ipv4.tcp_fastopen_blackhole_timeout_sec = 0
net.ipv4.tcp_fastopen_key = 00000000-00000000-00000000-00000000
net.ipv4.tcp_fin_timeout = 60
net.ipv4.tcp_frto = 2
net.ipv4.tcp_fwmark_accept = 0
net.ipv4.tcp_invalid_ratelimit = 500
net.ipv4.tcp_keepalive_intvl = 75
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_time = 7200
net.ipv4.tcp_l3mdev_accept = 0
net.ipv4.tcp_limit_output_bytes = 1048576
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_max_orphans = 32768
net.ipv4.tcp_max_reordering = 300
net.ipv4.tcp_max_syn_backlog = 512
net.ipv4.tcp_max_tw_buckets = 32768
net.ipv4.tcp_mem = 91272	121697	182544
net.ipv4.tcp_migrate_req = 0
net.ipv4.tcp_min_rtt_wlen = 300
net.ipv4.tcp_min_snd_mss = 48
net.ipv4.tcp_min_tso_segs = 2
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_mtu_probe_floor = 48
net.ipv4.tcp_mtu_probing = 0
net.ipv4.tcp_no_metrics_save = 0
net.ipv4.tcp_no_ssthresh_metrics_save = 1
net.ipv4.tcp_notsent_lowat = 4294967295
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_pacing_ca_ratio = 120
net.ipv4.tcp_pacing_ss_ratio = 200
net.ipv4.tcp_probe_interval = 600
net.ipv4.tcp_probe_threshold = 8
net.ipv4.tcp_recovery = 1
net.ipv4.tcp_reflect_tos = 0
net.ipv4.tcp_reordering = 3
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_rmem = 4096	131072	6291456
net.ipv4.tcp_sack = 1
net.ipv4.tcp_shrink_window = 0
net.ipv4.tcp_slow_start_after_idle = 1
net.ipv4.tcp_stdurg = 0
net.ipv4.tcp_syn_retries = 6
net.ipv4.tcp_synack_retries = 5
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_thin_linear_timeouts = 0
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_tso_rtt_log = 9
net.ipv4.tcp_tso_win_divisor = 3
net.ipv4.tcp_tw_reuse = 2
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_wmem = 4096	20480	4194304
net.ipv4.tcp_workaround_signed_windows = 0
net.ipv4.udp_early_demux = 1
net.ipv4.udp_l3mdev_accept = 0
net.ipv4.udp_mem = 182544	243394	365088
net.ipv4.udp_rmem_min = 4096
net.ipv4.udp_wmem_min = 4096
net.ipv4.xfrm4_gc_thresh = 32768
net.ipv6.anycast_src_echo_reply = 0
net.ipv6.auto_flowlabels = 1
net.ipv6.bindv6only = 0
net.ipv6.calipso_cache_bucket_size = 10
net.ipv6.calipso_cache_enable = 1
net.ipv6.conf.all.accept_dad = 0
net.ipv6.conf.all.accept_ra = 1
net.ipv6.conf.all.accept_ra_defrtr = 1
net.ipv6.conf.all.accept_ra_from_local = 0
net.ipv6.conf.all.accept_ra_min_hop_limit = 1
net.ipv6.conf.all.accept_ra_min_lft = 0
net.ipv6.conf.all.accept_ra_mtu = 1
net.ipv6.conf.all.accept_ra_pinfo = 1
net.ipv6.conf.all.accept_ra_rt_info_max_plen = 0
net.ipv6.conf.all.accept_ra_rt_info_min_plen = 0
net.ipv6.conf.all.accept_ra_rtr_pref = 1
net.ipv6.conf.all.accept_redirects = 1
net.ipv6.conf.all.accept_source_route = 0
net.ipv6.conf.all.accept_untracked_na = 0
net.ipv6.conf.all.addr_gen_mode = 0
net.ipv6.conf.all.autoconf = 1
net.ipv6.conf.all.dad_transmits = 1
net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.all.disable_policy = 0
net.ipv6.conf.all.drop_unicast_in_l2_multicast = 0
net.ipv6.conf.all.drop_unsolicited_na = 0
net.ipv6.conf.all.enhanced_dad = 1
net.ipv6.conf.all.force_mld_version = 0
net.ipv6.conf.all.force_tllao = 0
net.ipv6.conf.all.forwarding = 0
net.ipv6.conf.all.hop_limit = 64
net.ipv6.conf.all.ignore_routes_with_linkdown = 0
net.ipv6.conf.all.ioam6_enabled = 0
net.ipv6.conf.all.ioam6_id = 65535
net.ipv6.conf.all.ioam6_id_wide = 4294967295
net.ipv6.conf.all.keep_addr_on_down = 0
net.ipv6.conf.all.max_addresses = 16
net.ipv6.conf.all.max_desync_factor = 600
net.ipv6.conf.all.mc_forwarding = 0
net.ipv6.conf.all.mldv1_unsolicited_report_interval = 10000
net.ipv6.conf.all.mldv2_unsolicited_report_interval = 1000
net.ipv6.conf.all.mtu = 1280
net.ipv6.conf.all.ndisc_evict_nocarrier = 1
net.ipv6.conf.all.ndisc_notify = 0
net.ipv6.conf.all.ndisc_tclass = 0
net.ipv6.conf.all.optimistic_dad = 0
net.ipv6.conf.all.proxy_ndp = 0
net.ipv6.conf.all.ra_defrtr_metric = 1024
net.ipv6.conf.all.regen_max_retry = 3
net.ipv6.conf.all.router_probe_interval = 60
net.ipv6.conf.all.router_solicitation_delay = 1
net.ipv6.conf.all.router_solicitation_interval = 4
net.ipv6.conf.all.router_solicitation_max_interval = 3600
net.ipv6.conf.all.router_solicitations = -1
net.ipv6.conf.all.rpl_seg_enabled = 0
net.ipv6.conf.all.seg6_enabled = 0
net.ipv6.conf.all.seg6_require_hmac = 0
net.ipv6.conf.all.suppress_frag_ndisc = 1
net.ipv6.conf.all.temp_prefered_lft = 86400
net.ipv6.conf.all.temp_valid_lft = 604800
net.ipv6.conf.all.use_oif_addrs_only = 0
net.ipv6.conf.all.use_optimistic = 0
net.ipv6.conf.all.use_tempaddr = 0
net.ipv6.conf.default.accept_dad = 1
net.ipv6.conf.default.accept_ra = 1
net.ipv6.conf.default.accept_ra_defrtr = 1
net.ipv6.conf.default.accept_ra_from_local = 0
net.ipv6.conf.default.accept_ra_min_hop_limit = 1
net.ipv6.conf.default.accept_ra_min_lft = 0
net.ipv6.conf.default.accept_ra_mtu = 1
net.ipv6.conf.default.accept_ra_pinfo = 1
net.ipv6.conf.default.accept_ra_rt_info_max_plen = 0
net.ipv6.conf.default.accept_ra_rt_info_min_plen = 0
net.ipv6.conf.default.accept_ra_rtr_pref = 1
net.ipv6.conf.default.accept_redirects = 1
net.ipv6.conf.default.accept_source_route = 0
net.ipv6.conf.default.accept_untracked_na = 0
net.ipv6.conf.default.addr_gen_mode = 0
net.ipv6.conf.default.autoconf = 1
net.ipv6.conf.default.dad_transmits = 1
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.default.disable_policy = 0
net.ipv6.conf.default.drop_unicast_in_l2_multicast = 0
net.ipv6.conf.default.drop_unsolicited_na = 0
net.ipv6.conf.default.enhanced_dad = 1
net.ipv6.conf.default.force_mld_version = 0
net.ipv6.conf.default.force_tllao = 0
net.ipv6.conf.default.forwarding = 0
net.ipv6.conf.default.hop_limit = 64
net.ipv6.conf.default.ignore_routes_with_linkdown = 0
net.ipv6.conf.default.ioam6_enabled = 0
net.ipv6.conf.default.ioam6_id = 65535
net.ipv6.conf.default.ioam6_id_wide = 4294967295
net.ipv6.conf.default.keep_addr_on_down = 0
net.ipv6.conf.default.max_addresses = 16
net.ipv6.conf.default.max_desync_factor = 600
net.ipv6.conf.default.mc_forwarding = 0
net.ipv6.conf.default.mldv1_unsolicited_report_interval = 10000
net.ipv6.conf.default.mldv2_unsolicited_report_interval = 1000
net.ipv6.conf.default.mtu = 1280
net.ipv6.conf.default.ndisc_evict_nocarrier = 1
net.ipv6.conf.default.ndisc_notify = 0
net.ipv6.conf.default.ndisc_tclass = 0
net.ipv6.conf.default.optimistic_dad = 0
net.ipv6.conf.default.proxy_ndp = 0
net.ipv6.conf.default.ra_defrtr_metric = 1024
net.ipv6.conf.default.regen_max_retry = 3
net.ipv6.conf.default.router_probe_interval = 60
net.ipv6.conf.default.router_solicitation_delay = 1
net.ipv6.conf.default.router_solicitation_interval = 4
net.ipv6.conf.default.router_solicitation_max_interval = 3600
net.ipv6.conf.default.router_solicitations = -1
net.ipv6.conf.default.rpl_seg_enabled = 0
net.ipv6.conf.default.seg6_enabled = 0
net.ipv6.conf.default.seg6_require_hmac = 0
net.ipv6.conf.default.suppress_frag_ndisc = 1
net.ipv6.conf.default.temp_prefered_lft = 86400
net.ipv6.conf.default.temp_valid_lft = 604800
net.ipv6.conf.default.use_oif_addrs_only = 0
net.ipv6.conf.default.use_optimistic = 0
net.ipv6.conf.default.use_tempaddr = 0
net.ipv6.conf.enp39s0.accept_dad = 1
net.ipv6.conf.enp39s0.accept_ra = 0
net.ipv6.conf.enp39s0.accept_ra_defrtr = 1
net.ipv6.conf.enp39s0.accept_ra_from_local = 0
net.ipv6.conf.enp39s0.accept_ra_min_hop_limit = 1
net.ipv6.conf.enp39s0.accept_ra_min_lft = 0
net.ipv6.conf.enp39s0.accept_ra_mtu = 1
net.ipv6.conf.enp39s0.accept_ra_pinfo = 1
net.ipv6.conf.enp39s0.accept_ra_rt_info_max_plen = 0
net.ipv6.conf.enp39s0.accept_ra_rt_info_min_plen = 0
net.ipv6.conf.enp39s0.accept_ra_rtr_pref = 1
net.ipv6.conf.enp39s0.accept_redirects = 1
net.ipv6.conf.enp39s0.accept_source_route = 0
net.ipv6.conf.enp39s0.accept_untracked_na = 0
net.ipv6.conf.enp39s0.addr_gen_mode = 0
net.ipv6.conf.enp39s0.autoconf = 1
net.ipv6.conf.enp39s0.dad_transmits = 0
net.ipv6.conf.enp39s0.disable_ipv6 = 0
net.ipv6.conf.enp39s0.disable_policy = 0
net.ipv6.conf.enp39s0.drop_unicast_in_l2_multicast = 0
net.ipv6.conf.enp39s0.drop_unsolicited_na = 0
net.ipv6.conf.enp39s0.enhanced_dad = 1
net.ipv6.conf.enp39s0.force_mld_version = 0
net.ipv6.conf.enp39s0.force_tllao = 0
net.ipv6.conf.enp39s0.forwarding = 0
net.ipv6.conf.enp39s0.hop_limit = 64
net.ipv6.conf.enp39s0.ignore_routes_with_linkdown = 0
net.ipv6.conf.enp39s0.ioam6_enabled = 0
net.ipv6.conf.enp39s0.ioam6_id = 65535
net.ipv6.conf.enp39s0.ioam6_id_wide = 4294967295
net.ipv6.conf.enp39s0.keep_addr_on_down = 0
net.ipv6.conf.enp39s0.max_addresses = 16
net.ipv6.conf.enp39s0.max_desync_factor = 600
net.ipv6.conf.enp39s0.mc_forwarding = 0
net.ipv6.conf.enp39s0.mldv1_unsolicited_report_interval = 10000
net.ipv6.conf.enp39s0.mldv2_unsolicited_report_interval = 1000
net.ipv6.conf.enp39s0.mtu = 9001
net.ipv6.conf.enp39s0.ndisc_evict_nocarrier = 1
net.ipv6.conf.enp39s0.ndisc_notify = 0
net.ipv6.conf.enp39s0.ndisc_tclass = 0
net.ipv6.conf.enp39s0.optimistic_dad = 0
net.ipv6.conf.enp39s0.proxy_ndp = 0
net.ipv6.conf.enp39s0.ra_defrtr_metric = 1024
net.ipv6.conf.enp39s0.regen_max_retry = 3
net.ipv6.conf.enp39s0.router_probe_interval = 60
net.ipv6.conf.enp39s0.router_solicitation_delay = 1
net.ipv6.conf.enp39s0.router_solicitation_interval = 4
net.ipv6.conf.enp39s0.router_solicitation_max_interval = 3600
net.ipv6.conf.enp39s0.router_solicitations = -1
net.ipv6.conf.enp39s0.rpl_seg_enabled = 0
net.ipv6.conf.enp39s0.seg6_enabled = 0
net.ipv6.conf.enp39s0.seg6_require_hmac = 0
net.ipv6.conf.enp39s0.suppress_frag_ndisc = 1
net.ipv6.conf.enp39s0.temp_prefered_lft = 86400
net.ipv6.conf.enp39s0.temp_valid_lft = 604800
net.ipv6.conf.enp39s0.use_oif_addrs_only = 0
net.ipv6.conf.enp39s0.use_optimistic = 0
net.ipv6.conf.enp39s0.use_tempaddr = 0
net.ipv6.conf.lo.accept_dad = -1
net.ipv6.conf.lo.accept_ra = 1
net.ipv6.conf.lo.accept_ra_defrtr = 1
net.ipv6.conf.lo.accept_ra_from_local = 0
net.ipv6.conf.lo.accept_ra_min_hop_limit = 1
net.ipv6.conf.lo.accept_ra_min_lft = 0
net.ipv6.conf.lo.accept_ra_mtu = 1
net.ipv6.conf.lo.accept_ra_pinfo = 1
net.ipv6.conf.lo.accept_ra_rt_info_max_plen = 0
net.ipv6.conf.lo.accept_ra_rt_info_min_plen = 0
net.ipv6.conf.lo.accept_ra_rtr_pref = 1
net.ipv6.conf.lo.accept_redirects = 1
net.ipv6.conf.lo.accept_source_route = 0
net.ipv6.conf.lo.accept_untracked_na = 0
net.ipv6.conf.lo.addr_gen_mode = 0
net.ipv6.conf.lo.autoconf = 1
net.ipv6.conf.lo.dad_transmits = 1
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.lo.disable_policy = 0
net.ipv6.conf.lo.drop_unicast_in_l2_multicast = 0
net.ipv6.conf.lo.drop_unsolicited_na = 0
net.ipv6.conf.lo.enhanced_dad = 1
net.ipv6.conf.lo.force_mld_version = 0
net.ipv6.conf.lo.force_tllao = 0
net.ipv6.conf.lo.forwarding = 0
net.ipv6.conf.lo.hop_limit = 64
net.ipv6.conf.lo.ignore_routes_with_linkdown = 0
net.ipv6.conf.lo.ioam6_enabled = 0
net.ipv6.conf.lo.ioam6_id = 65535
net.ipv6.conf.lo.ioam6_id_wide = 4294967295
net.ipv6.conf.lo.keep_addr_on_down = 0
net.ipv6.conf.lo.max_addresses = 16
net.ipv6.conf.lo.max_desync_factor = 600
net.ipv6.conf.lo.mc_forwarding = 0
net.ipv6.conf.lo.mldv1_unsolicited_report_interval = 10000
net.ipv6.conf.lo.mldv2_unsolicited_report_interval = 1000
net.ipv6.conf.lo.mtu = 65536
net.ipv6.conf.lo.ndisc_evict_nocarrier = 1
net.ipv6.conf.lo.ndisc_notify = 0
net.ipv6.conf.lo.ndisc_tclass = 0
net.ipv6.conf.lo.optimistic_dad = 0
net.ipv6.conf.lo.proxy_ndp = 0
net.ipv6.conf.lo.ra_defrtr_metric = 1024
net.ipv6.conf.lo.regen_max_retry = 3
net.ipv6.conf.lo.router_probe_interval = 60
net.ipv6.conf.lo.router_solicitation_delay = 1
net.ipv6.conf.lo.router_solicitation_interval = 4
net.ipv6.conf.lo.router_solicitation_max_interval = 3600
net.ipv6.conf.lo.router_solicitations = -1
net.ipv6.conf.lo.rpl_seg_enabled = 0
net.ipv6.conf.lo.seg6_enabled = 0
net.ipv6.conf.lo.seg6_require_hmac = 0
net.ipv6.conf.lo.suppress_frag_ndisc = 1
net.ipv6.conf.lo.temp_prefered_lft = 86400
net.ipv6.conf.lo.temp_valid_lft = 604800
net.ipv6.conf.lo.use_oif_addrs_only = 0
net.ipv6.conf.lo.use_optimistic = 0
net.ipv6.conf.lo.use_tempaddr = -1
net.ipv6.fib_multipath_hash_fields = 7
net.ipv6.fib_multipath_hash_policy = 0
net.ipv6.fib_notify_on_flag_change = 0
net.ipv6.flowlabel_consistency = 1
net.ipv6.flowlabel_reflect = 0
net.ipv6.flowlabel_state_ranges = 0
net.ipv6.fwmark_reflect = 0
net.ipv6.icmp.echo_ignore_all = 0
net.ipv6.icmp.echo_ignore_anycast = 0
net.ipv6.icmp.echo_ignore_multicast = 0
net.ipv6.icmp.ratelimit = 1000
net.ipv6.icmp.ratemask = 0-1,3-127
net.ipv6.idgen_delay = 1
net.ipv6.idgen_retries = 3
net.ipv6.ioam6_id = 16777215
net.ipv6.ioam6_id_wide = 72057594037927935
net.ipv6.ip6frag_high_thresh = 4194304
net.ipv6.ip6frag_low_thresh = 3145728
net.ipv6.ip6frag_secret_interval = 0
net.ipv6.ip6frag_time = 60
net.ipv6.ip_nonlocal_bind = 0
net.ipv6.max_dst_opts_length = 2147483647
net.ipv6.max_dst_opts_number = 8
net.ipv6.max_hbh_length = 2147483647
net.ipv6.max_hbh_opts_number = 8
net.ipv6.mld_max_msf = 64
net.ipv6.mld_qrv = 2
net.ipv6.neigh.default.anycast_delay = 100
net.ipv6.neigh.default.app_solicit = 0
net.ipv6.neigh.default.base_reachable_time_ms = 30000
net.ipv6.neigh.default.delay_first_probe_time = 5
net.ipv6.neigh.default.gc_interval = 30
net.ipv6.neigh.default.gc_stale_time = 60
net.ipv6.neigh.default.gc_thresh1 = 0
net.ipv6.neigh.default.gc_thresh2 = 15360
net.ipv6.neigh.default.gc_thresh3 = 16384
net.ipv6.neigh.default.interval_probe_time_ms = 5000
net.ipv6.neigh.default.locktime = 0
net.ipv6.neigh.default.mcast_resolicit = 0
net.ipv6.neigh.default.mcast_solicit = 3
net.ipv6.neigh.default.proxy_delay = 80
net.ipv6.neigh.default.proxy_qlen = 64
net.ipv6.neigh.default.retrans_time_ms = 1000
net.ipv6.neigh.default.ucast_solicit = 3
net.ipv6.neigh.default.unres_qlen = 101
net.ipv6.neigh.default.unres_qlen_bytes = 212992
net.ipv6.neigh.enp39s0.anycast_delay = 100
net.ipv6.neigh.enp39s0.app_solicit = 0
net.ipv6.neigh.enp39s0.base_reachable_time_ms = 30000
net.ipv6.neigh.enp39s0.delay_first_probe_time = 5
net.ipv6.neigh.enp39s0.gc_stale_time = 60
net.ipv6.neigh.enp39s0.interval_probe_time_ms = 5000
net.ipv6.neigh.enp39s0.locktime = 0
net.ipv6.neigh.enp39s0.mcast_resolicit = 0
net.ipv6.neigh.enp39s0.mcast_solicit = 3
net.ipv6.neigh.enp39s0.proxy_delay = 80
net.ipv6.neigh.enp39s0.proxy_qlen = 64
net.ipv6.neigh.enp39s0.retrans_time_ms = 1000
net.ipv6.neigh.enp39s0.ucast_solicit = 3
net.ipv6.neigh.enp39s0.unres_qlen = 101
net.ipv6.neigh.enp39s0.unres_qlen_bytes = 212992
net.ipv6.neigh.lo.anycast_delay = 100
net.ipv6.neigh.lo.app_solicit = 0
net.ipv6.neigh.lo.base_reachable_time_ms = 30000
net.ipv6.neigh.lo.delay_first_probe_time = 5
net.ipv6.neigh.lo.gc_stale_time = 60
net.ipv6.neigh.lo.interval_probe_time_ms = 5000
net.ipv6.neigh.lo.locktime = 0
net.ipv6.neigh.lo.mcast_resolicit = 0
net.ipv6.neigh.lo.mcast_solicit = 3
net.ipv6.neigh.lo.proxy_delay = 80
net.ipv6.neigh.lo.proxy_qlen = 64
net.ipv6.neigh.lo.retrans_time_ms = 1000
net.ipv6.neigh.lo.ucast_solicit = 3
net.ipv6.neigh.lo.unres_qlen = 101
net.ipv6.neigh.lo.unres_qlen_bytes = 212992
net.ipv6.route.gc_elasticity = 9
net.ipv6.route.gc_interval = 30
net.ipv6.route.gc_min_interval = 0
net.ipv6.route.gc_min_interval_ms = 500
net.ipv6.route.gc_thresh = 1024
net.ipv6.route.gc_timeout = 60
net.ipv6.route.max_size = 2147483647
net.ipv6.route.min_adv_mss = 1220
net.ipv6.route.mtu_expires = 600
net.ipv6.route.skip_notify_on_dev_down = 0
net.ipv6.seg6_flowlabel = 0
net.ipv6.xfrm6_gc_thresh = 32768
net.mptcp.add_addr_timeout = 120
net.mptcp.allow_join_initial_addr_port = 1
net.mptcp.checksum_enabled = 0
net.mptcp.enabled = 1
net.mptcp.pm_type = 0
net.mptcp.stale_loss_cnt = 4
net.netfilter.nf_log.0 = NONE
net.netfilter.nf_log.1 = NONE
net.netfilter.nf_log.10 = NONE
net.netfilter.nf_log.2 = NONE
net.netfilter.nf_log.3 = NONE
net.netfilter.nf_log.4 = NONE
net.netfilter.nf_log.5 = NONE
net.netfilter.nf_log.6 = NONE
net.netfilter.nf_log.7 = NONE
net.netfilter.nf_log.8 = NONE
net.netfilter.nf_log.9 = NONE
net.netfilter.nf_log_all_netns = 0
net.unix.max_dgram_qlen = 512
sunrpc.max_resvport = 1023
sunrpc.min_resvport = 665
sunrpc.nfs_debug = 0x0000
sunrpc.nfsd_debug = 0x0000
sunrpc.nlm_debug = 0x0000
sunrpc.rpc_debug = 0x0000
sunrpc.tcp_fin_timeout = 15
sunrpc.tcp_max_slot_table_entries = 65536
sunrpc.tcp_slot_table_entries = 2
sunrpc.transports = tcp 1048576
sunrpc.transports = udp 32768
sunrpc.udp_slot_table_entries = 16
user.max_cgroup_namespaces = 30820
user.max_fanotify_groups = 128
user.max_fanotify_marks = 64735
user.max_inotify_instances = 128
user.max_inotify_watches = 60835
user.max_ipc_namespaces = 30820
user.max_mnt_namespaces = 30820
user.max_net_namespaces = 30820
user.max_pid_namespaces = 30820
user.max_time_namespaces = 30820
user.max_user_namespaces = 30820
user.max_uts_namespaces = 30820
vm.admin_reserve_kbytes = 8192
vm.compact_unevictable_allowed = 1
vm.compaction_proactiveness = 20
vm.dirty_background_bytes = 0
vm.dirty_background_ratio = 10
vm.dirty_bytes = 0
vm.dirty_expire_centisecs = 3000
vm.dirty_ratio = 20
vm.dirty_writeback_centisecs = 500
vm.dirtytime_expire_seconds = 43200
vm.extfrag_threshold = 500
vm.hugetlb_optimize_vmemmap = 0
vm.hugetlb_shm_group = 0
vm.laptop_mode = 0
vm.legacy_va_layout = 0
vm.lowmem_reserve_ratio = 256	256	32	0	0
vm.max_map_count = 65530
vm.memory_failure_early_kill = 0
vm.memory_failure_recovery = 1
vm.min_free_kbytes = 67584
vm.min_slab_ratio = 5
vm.min_unmapped_ratio = 1
vm.mmap_min_addr = 65536
vm.mmap_rnd_bits = 28
vm.mmap_rnd_compat_bits = 8
vm.nr_hugepages = 0
vm.nr_hugepages_mempolicy = 0
vm.nr_overcommit_hugepages = 0
vm.numa_stat = 1
vm.numa_zonelist_order = Node
vm.oom_dump_tasks = 1
vm.oom_kill_allocating_task = 0
vm.overcommit_kbytes = 0
vm.overcommit_memory = 0
vm.overcommit_ratio = 50
vm.page-cluster = 3
vm.page_lock_unfairness = 5
vm.panic_on_oom = 0
vm.percpu_pagelist_high_fraction = 0
vm.stat_interval = 1
vm.swappiness = 60
vm.unprivileged_userfaultfd = 0
vm.user_reserve_kbytes = 131072
vm.vfs_cache_pressure = 100
vm.watermark_boost_factor = 15000
vm.watermark_scale_factor = 10
vm.zone_reclaim_mode = 0