I tested this in isolation on a Bionic VM and here is what I see:

1) nf_conntrack_max is applied upon reboot if an entry in /etc/modules and there is an entry in one of the files under /etc/sysctl.d/;
2) systemd-sysctl service is the one applying those settings. It is made to run after the systemd-modules-load unit (`After=systemd-modules-load.service`).

https://github.com/systemd/systemd/blob/v237/units/systemd-sysctl.service.in#L15 (upstream)
https://github.com/systemd/systemd/commit/0b73eab7a2185ae0377650e3fdb8208347a8a575 (original commit)
https://git.launchpad.net/ubuntu/+source/systemd/tree/units/systemd-sysctl.service.in?h=ubuntu/bionic-updates#n15 (bionic-updates)


3) Both systemd-modules-load and systemd-sysctl run as a part of the sysinit.target - so very early in the boot process.

https://www.freedesktop.org/software/systemd/man/bootup.html#System%20Manager%20Bootup


Could you provide more information about the status of `systemd-modules-load` and `systemd-sysctl` units: i.e. when they ran and whether the systemd-sysctl failed? Maybe something else is overriding those settings instead?



➜  ~ lxc launch ubuntu:bionic ct-bionic --vm
# enable LXD agent ... https://discuss.linuxcontainers.org/t/running-virtual-machines-with-lxd-4-0/7519

➜  ~ lxc exec ct-bionic bash

root@ct-bionic:~# modprobe nf_conntrack

root@ct-bionic:~# sysctl net.nf_conntrack_max
net.nf_conntrack_max = 32768

root@ct-bionic:~# echo nf_conntrack >> /etc/modules
root@ct-bionic:~# echo 'net.nf_conntrack_max = 42424242' > /etc/sysctl.d/10-conntrack.conf
root@ct-bionic:~# sysctl -p /etc/sysctl.d/10-conntrack.conf
net.nf_conntrack_max = 42424242

root@ct-bionic:~# reboot
# exec again

root@ct-bionic:~# lsmod | grep conntrack
nf_conntrack          135168  0

root@ct-bionic:~# sysctl net.nf_conntrack_max
net.nf_conntrack_max = 42424242


root@ct-bionic:~# sudo systemctl list-dependencies 
default.target
● ├─accounts-daemon.service
● ├─apport.service
● ├─display-manager.service
● ├─grub-common.service
● ├─systemd-update-utmp-runlevel.service
● ├─ureadahead.service
● └─multi-user.target
# ...
●   ├─basic.target
# ...
●   │ ├─sysinit.target
# ...
●   │ │ ├─systemd-machine-id-commit.service
●   │ │ ├─systemd-modules-load.service
●   │ │ ├─systemd-random-seed.service
●   │ │ ├─systemd-sysctl.service


root@ct-bionic:~# systemctl cat systemd-sysctl.service
[Unit]
Description=Apply Kernel Variables
Documentation=man:systemd-sysctl.service(8) man:sysctl.d(5)
DefaultDependencies=no
Conflicts=shutdown.target
After=systemd-modules-load.service               # <----- this
Before=sysinit.target shutdown.target
ConditionPathIsReadWrite=/proc/sys/net/

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/lib/systemd/systemd-sysctl
TimeoutSec=90s


root@ct-bionic:~# systemctl status systemd-sysctl
● systemd-sysctl.service - Apply Kernel Variables
   Loaded: loaded (/lib/systemd/system/systemd-sysctl.service; static; vendor preset: enabled)
   Active: active (exited) since Mon 2021-05-24 09:53:27 UTC; 28min ago
     Docs: man:systemd-sysctl.service(8)
           man:sysctl.d(5)
  Process: 482 ExecStart=/lib/systemd/systemd-sysctl (code=exited, status=0/SUCCESS)
 Main PID: 482 (code=exited, status=0/SUCCESS)

root@ct-bionic:~# systemctl status systemd-modules-load.service
● systemd-modules-load.service - Load Kernel Modules
   Loaded: loaded (/lib/systemd/system/systemd-modules-load.service; static; vendor preset: enabled)
   Active: active (exited) since Mon 2021-05-24 09:53:27 UTC; 33min ago
     Docs: man:systemd-modules-load.service(8)
           man:modules-load.d(5)
  Process: 430 ExecStart=/lib/systemd/systemd-modules-load (code=exited, status=0/SUCCESS)
 Main PID: 430 (code=exited, status=0/SUCCESS)