This is a follow-up bug to https://bugs.launchpad.net/ubuntu/+source/linux-aws-5.13/+bug/1977919
I can confirm that the problem is indeed not fully fixed. @electricdaemon said:
> Test kernel posted fixes crash but has another bug with unkillable stuck defunct docker-proxy service causing more issues. Bug is not solved. Tested on Linux AWS Lightsail instance.
What I'm seeing is that docker-compose stacks either don't start at all or only start partially. In both cases the affected containers cannot start due to their host port being already allocated. I can say with absolute certainty that the ports on the host are dedicated to container applications and no other service is actually bound to the affected port numbers.
# uname -a
Linux ip-10-0-69-193 5.13.0-1029-aws #32~20.04.1-Ubuntu SMP Thu Jun 9 13:03:13 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
# apt-cache policy docker containerd
docker:
Installed: (none)
Candidate: 1.5-2
Version table:
1.5-2 500
500 http://eu-central-1.ec2.archive.ubuntu.com/ubuntu focal/universe amd64 Packages
containerd:
Installed: (none)
Candidate: 1.5.9-0ubuntu1~20.04.4
Version table:
1.5.9-0ubuntu1~20.04.4 500
500 http://eu-central-1.ec2.archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages
500 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages
1.3.3-0ubuntu2 500
500 http://eu-central-1.ec2.archive.ubuntu.com/ubuntu focal/main amd64 Packages
# docker-compose --version
docker-compose version 1.29.2, build 5becea4c
root@ip-10-0-69-193:/opt/myapp8/myappserv/int# docker-compose up -d
Creating network "myappserv-int_default" with the default driver
Creating myapp-migrator-int ... done
Creating myapp-dealer-int ...
Creating myapp-offer-int ...
Creating myapp-customer-int ...
Creating myapp-customer-int ... error
Creating myapp-dealer-int ... done
Creating myapp-offer-int ... done
: port is already allocated
ERROR: for customer Cannot start service customer: driver failed programming external connectivity on endpoint myapp8-customer-int (fe4112364528b0e7d192c793929c579e8a81af715118c8f83ad7e65e7397f3be): Bind for 0.0.0.0:9001 failed: port is already allocated
ERROR: Encountered errors while bringing up the project.
root@ip-10-0-69-193:/opt/myapp8/myappserv/int# docker-compose down
Stopping myapp8-offer-int ... done
Stopping myapp8-dealer-int ... done
Removing myapp8-customer-int ... done
Removing myapp8-offer-int ... done
Removing myapp8-dealer-int ... done
Removing myapp8-migrator-int ... done
Removing network myappserv-int_default
root@ip-10-0-69-193:/opt/myapp8/myappserv/int# docker-compose up -d
Creating network "myappserv-int_default" with the default driver
Creating myapp8-migrator-int ... done
Creating myapp8-offer-int ...
Creating myapp8-customer-int ...
Creating myapp8-customer-int ... error
WARNING: Host is already in use by another container
Creating myapp8-offer-int ... done
ERROR: for myapp8-customer-int Cannot start service customer: driver failed programming external connectivity on endpoint myapp8-customer-int (72fc08854cd278e63cd3234e7fb03c08cb045efdcfb9e42075a1250d893645d5): Bind for 0.0.0.0:9001 failed
Creating myapp8-dealer-int ... done
ERROR: for customer Cannot start service customer: driver failed programming external connectivity on endpoint myapp8-customer-int (72fc08854cd278e63cd3234e7fb03c08cb045efdcfb9e42075a1250d893645d5): Bind for 0.0.0.0:9001 failed: port is already allocated
ERROR: Encountered errors while bringing up the project.
# docker-compose config
services:
customer:
container_name: myapp8-customer-int
depends_on:
migrator:
condition: service_completed_successfully
image: reg.mydomain.tld/myapp8/customer:430d4ca
ports:
- published: 9001
target: 9001
restart: always
dealer:
container_name: myapp8-dealer-int
depends_on:
migrator:
condition: service_completed_successfully
image: reg.mydomain.tld/myapp8/dealer:430d4ca
ports:
- published: 9002
target: 9002
restart: always
migrator:
container_name: myapp8-migrator-int
image: reg.mydomain.tld/myapp8/migrator:430d4ca
offer:
container_name: myapp8-offer-int
depends_on:
migrator:
condition: service_completed_successfully
image: reg.mydomain.tld/myapp8/offer:430d4ca
ports:
- published: 9003
target: 9003
restart: always
version: '3'
# ps aux | grep docker-proxy
root 997 0.0 0.0 1075148 3552 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip 0.0.0.0 -host-port 19000 -container-ip 172.21.0.2 -container-port 9000
root 1003 0.0 0.0 1148624 3756 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip :: -host-port 19000 -container-ip 172.21.0.2 -container-port 9000
root 1016 0.0 0.0 1148880 3716 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip 0.0.0.0 -host-port 8065 -container-ip 172.27.0.2 -container-port 8055
root 1022 0.0 0.0 1222356 3612 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip :: -host-port 8065 -container-ip 172.27.0.2 -container-port 8055
root 1037 0.0 0.0 1222612 3640 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip 0.0.0.0 -host-port 8055 -container-ip 172.23.0.2 -container-port 8055
root 1043 0.0 0.0 1075148 3584 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip :: -host-port 8055 -container-ip 172.23.0.2 -container-port 8055
root 1077 0.0 0.0 1148880 3640 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip 127.0.0.1 -host-port 40000 -container-ip 172.18.0.2 -container-port 80
root 1090 0.0 0.0 1148880 4140 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip 0.0.0.0 -host-port 9001 -container-ip 172.26.0.4 -container-port 9001
root 1096 0.0 0.0 1148624 3588 ? Sl 08:18 0:00 /usr/bin/docker-proxy -proto tcp -host-ip :: -host-port 9001 -container-ip 172.26.0.4 -container-port 9001
root 4519 0.0 0.0 1222612 3896 ? Sl 09:00 0:00 /usr/bin/docker-proxy -proto tcp -host-ip 0.0.0.0 -host-port 9002 -container-ip 172.28.0.3 -container-port 9002
root 4525 0.0 0.0 1074892 3644 ? Sl 09:00 0:00 /usr/bin/docker-proxy -proto tcp -host-ip :: -host-port 9002 -container-ip 172.28.0.3 -container-port 9002
root 4539 0.0 0.0 1148880 3716 ? Sl 09:00 0:00 /usr/bin/docker-proxy -proto tcp -host-ip 0.0.0.0 -host-port 9003 -container-ip 172.28.0.2 -container-port 9003
root 4544 0.0 0.0 1074892 3740 ? Sl 09:00 0:00 /usr/bin/docker-proxy -proto tcp -host-ip :: -host-port 9003 -container-ip 172.28.0.2 -container-port 9003
# netstat -tulpn | egrep "(Foreign|docker-proxy)"
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:9001 0.0.0.0:* LISTEN 1090/docker-proxy
tcp 0 0 0.0.0.0:9002 0.0.0.0:* LISTEN 4519/docker-proxy
tcp 0 0 0.0.0.0:9003 0.0.0.0:* LISTEN 4539/docker-proxy
tcp 0 0 0.0.0.0:8055 0.0.0.0:* LISTEN 1037/docker-proxy
tcp 0 0 0.0.0.0:19000 0.0.0.0:* LISTEN 997/docker-proxy
tcp 0 0 127.0.0.1:40000 0.0.0.0:* LISTEN 1077/docker-proxy
tcp 0 0 0.0.0.0:8065 0.0.0.0:* LISTEN 1016/docker-proxy
tcp6 0 0 :::9001 :::* LISTEN 1096/docker-proxy
tcp6 0 0 :::9002 :::* LISTEN 4525/docker-proxy
tcp6 0 0 :::9003 :::* LISTEN 4544/docker-proxy
tcp6 0 0 :::8055 :::* LISTEN 1043/docker-proxy
tcp6 0 0 :::19000 :::* LISTEN 1003/docker-proxy
tcp6 0 0 :::8065 :::* LISTEN 1022/docker-proxy
Docker daemon startup log with port binding issues
-- Reboot --
Jun 13 08:18:27 ip-10-0-69-193 systemd[1]: Starting Docker Application Container Engine...
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.256199431Z" level=info msg="Starting up"
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.259890222Z" level=info msg="detected 127.0.0.53 nameserver, assuming systemd-resolved, so using resolv.conf: /run/systemd/resolve/resolv.conf"
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.338986729Z" level=info msg="parsed scheme: \"unix\"" module=grpc
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.339028163Z" level=info msg="scheme \"unix\" not registered, fallback to default scheme" module=grpc
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.339788195Z" level=info msg="ccResolverWrapper: sending update to cc: {[{unix:///run/containerd/containerd.sock <nil> 0 <nil>}] <nil> <nil>}" module=grpc
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.339818518Z" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.370153265Z" level=info msg="parsed scheme: \"unix\"" module=grpc
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.373318484Z" level=info msg="scheme \"unix\" not registered, fallback to default scheme" module=grpc
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.373508320Z" level=info msg="ccResolverWrapper: sending update to cc: {[{unix:///run/containerd/containerd.sock <nil> 0 <nil>}] <nil> <nil>}" module=grpc
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.373689058Z" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
Jun 13 08:18:28 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:28.521565874Z" level=info msg="[graphdriver] using prior storage driver: overlay2"
Jun 13 08:18:29 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:29.562866065Z" level=warning msg="Your kernel does not support CPU realtime scheduler"
Jun 13 08:18:29 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:29.563318522Z" level=warning msg="Your kernel does not support cgroup blkio weight"
Jun 13 08:18:29 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:29.563408595Z" level=warning msg="Your kernel does not support cgroup blkio weight_device"
Jun 13 08:18:29 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:29.563993369Z" level=info msg="Loading containers: start."
Jun 13 08:18:30 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:30.607949827Z" level=info msg="Removing stale sandbox 3b05224a2aacde133ba6e5b6b38e5958caca1cd1e25c27a4fc927fe9b0d0e64f (830dd3e1f0f166cd196e6ba7ce968331c9b54a78418cfe94411ffd29b42a2da2)"
Jun 13 08:18:30 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:30.632726976Z" level=warning msg="Error (Unable to complete atomic operation, key modified) deleting object [endpoint 7ee2dcd248f6607a560671a13f4938550bf4640565a589da68982a00817caa6f d4aa1d1b46f9d60e4c63b26d7403860f753d88791989e7a67538846016a0780b], retrying...."
Jun 13 08:18:30 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:30.791435745Z" level=info msg="Removing stale sandbox 887458aa16f64a4a07772ff6d5e154a271b73a6629704134a7c2b713bbd6d565 (eb1fb5718de82dd7719597e5cbac1091159ce4e94379dc07b6ebecfcd74d586e)"
Jun 13 08:18:30 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:30.927944615Z" level=info msg="Removing stale sandbox c73087bb5c2be5f386b2fbaef1d2594be14939e77ca63d95cd9dbe9d62e70ba9 (734e7cab764e533809b6edb6b3f7bdaef174651411e2011107cf4c55c45c8170)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.056824971Z" level=info msg="Removing stale sandbox e8148d1ffec21c1f44d2a24be5bb6c1d0c7b8c91998ff1e9bb5aa7bebe4ca6e3 (0bd839cbe4ebc01328a9ec8368395eadda7e72ffeffbfed42540a51dea68feca)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.342728937Z" level=info msg="Removing stale sandbox 0888ff9891abfe4955d610ca1f52c898553b2ef03c05bb510cc837282ca47711 (5a8feb357508e72d55e8b38231e34ecd90f8c2dc596756637e8b0c023ae63369)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.480517319Z" level=info msg="Removing stale sandbox 0c00e3eb1d3fb560231f3cfc4dd6d43b6128e82a4303e048bc3e1bce095c37e4 (ac811048aac5694c80d44bd8feff50e2baf0f7c94fa30331d93f90446960be93)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.632098665Z" level=info msg="Removing stale sandbox 210c64a62ea5bdd3d0d11039971318520c027e8af4ed7223695b469a7fa91870 (b699d29d6699dc6d25507f9b58830c614d836903ecf49ae292605021c1692c00)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.638488221Z" level=info msg="Removing stale endpoint cms-int-dummy (1d3cb1bc1a4c628af3141e00a8b67767501c53e9df41583332917985cd7aa62d)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.643475450Z" level=info msg="Removing stale endpoint cms-int-contoso (ba362dceaab5a016e32a122131e933ed2c3c4927901726dc610e0d0be26acdde)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.647756569Z" level=info msg="Removing stale endpoint portainer (18ee707396ada0df1f20b4ac0c7f2f3dad34142071b65861bfc4f921387560f8)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.659188980Z" level=info msg="Removing stale endpoint logspout-forwarder_logspout_1 (86553ec65469cdcc5d40be071a8975b7dfc2295c4ccf97023dd45bb4e284063c)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.663687874Z" level=info msg="Removing stale endpoint promtail_promtail_1 (8f0cb821a7322a7f7390c093a538e4af595c042aeb59af0423ae501c1e66cc63)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.667646467Z" level=info msg="Removing stale endpoint mapp8-customer-int (e826dee7b0d1dc6b015b563e4ebe94169d8bc36cbf57f97bdc808329677c8957)"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.727213355Z" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.847480135Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.847673088Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.867425295Z" level=warning msg="Failed to allocate and map port 40000-40000: Bind for 127.0.0.1:40000 failed: port is already allocated"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.903819097Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.903850582Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.961822116Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.961877774Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:18:31 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:31.967810851Z" level=warning msg="Failed to allocate and map port 8065-8065: Bind for 0.0.0.0:8065 failed: port is already allocated"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.020268138Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.021928936Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.025449737Z" level=warning msg="Failed to allocate and map port 19000-19000: Bind for 0.0.0.0:19000 failed: port is already allocated"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.067237638Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.067272652Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.193806632Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.194391464Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.201922365Z" level=warning msg="Failed to allocate and map port 8055-8055: Bind for 0.0.0.0:8055 failed: port is already allocated"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.261898918Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.261925476Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.274600075Z" level=error msg="Container not cleaned up from containerd from previous run" container=eb1fb5718de82dd7719597e5cbac1091159ce4e94379dc07b6ebecfcd74d586e error="id already in use"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.278680667Z" level=error msg="failed to start container" container=734e7cab764e533809b6edb6b3f7bdaef174651411e2011107cf4c55c45c8170 error="driver failed programming external connectivity on endpoint cms-int-dummy (140d12526fd6ac50af276e5e4bfd9ddf58ddaefe08fb2212e491c252aee9a1eb): Bind for 0.0.0.0:8065 >
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.280102302Z" level=error msg="failed to start container" container=5a8feb357508e72d55e8b38231e34ecd90f8c2dc596756637e8b0c023ae63369 error="driver failed programming external connectivity on endpoint portainer (4fef35183d5abc13788f16520cb2128b19e5c7568f28f90ed68a6e01a0672856): Bind for 0.0.0.0:19000 failed: >
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.281626508Z" level=error msg="failed to start container" container=b699d29d6699dc6d25507f9b58830c614d836903ecf49ae292605021c1692c00 error="driver failed programming external connectivity on endpoint logspout-forwarder_logspout_1 (3d12e7dee4b448e4b8ab8c0d680bfa72094d3b6fdec7accfcf02e902f33fcfc7): Bind for 12>
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.324772875Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.324804948Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.375847635Z" level=error msg="failed to start container" container=0bd839cbe4ebc01328a9ec8368395eadda7e72ffeffbfed42540a51dea68feca error="driver failed programming external connectivity on endpoint cms-int-contoso (81163f037821378437ba958ee1efb1062467a678dc2570106ac9b8213bcedfc5): Bind for 0.0.0.0:8055>
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.437882668Z" level=warning msg="Failed to allocate and map port 9001-9001: Bind for 0.0.0.0:9001 failed: port is already allocated"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.541567238Z" level=error msg="Container not cleaned up from containerd from previous run" container=830dd3e1f0f166cd196e6ba7ce968331c9b54a78418cfe94411ffd29b42a2da2 error="id already in use"
Jun 13 08:18:32 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:32.634683839Z" level=error msg="failed to start container" container=ac811048aac5694c80d44bd8feff50e2baf0f7c94fa30331d93f90446960be93 error="driver failed programming external connectivity on endpoint mapp8-customer-int (f6187117263ef52b444366c3768fe5d6d2f790b5c2c67f92ebbb37ee95c3efb1): Bind for 0.0.0.0:9001>
Jun 13 08:18:33 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:33.634918366Z" level=info msg="Loading containers: done."
Jun 13 08:18:33 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:33.706060624Z" level=info msg="Docker daemon" commit=a89b842 graphdriver(s)=overlay2 version=20.10.17
Jun 13 08:18:33 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:33.707922063Z" level=info msg="Daemon has completed initialization"
Jun 13 08:18:33 ip-10-0-69-193 systemd[1]: Started Docker Application Container Engine.
Jun 13 08:18:33 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:18:33.862505842Z" level=info msg="API listen on /run/docker.sock"
Jun 13 08:22:43 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:43.789841985Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:22:43 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:43.790506837Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:22:45 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:45.185275300Z" level=info msg="ignoring event" container=4027e2353a12d39b7c26185341420e592a7df6422a566a031f4edef00a8ac774 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:22:45 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:45.381127409Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:22:45 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:45.381181456Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:22:45 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:45.387012769Z" level=warning msg="Failed to allocate and map port 9001-9001: Bind for 0.0.0.0:9001 failed: port is already allocated"
Jun 13 08:22:45 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:45.453286174Z" level=error msg="ac811048aac5694c80d44bd8feff50e2baf0f7c94fa30331d93f90446960be93 cleanup: failed to delete container from containerd: no such container"
Jun 13 08:22:45 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:45.453349031Z" level=error msg="Handler for POST /v1.41/containers/ac811048aac5694c80d44bd8feff50e2baf0f7c94fa30331d93f90446960be93/start returned error: driver failed programming external connectivity on endpoint mapp8-customer-int (7fa7459091a2af4e51957c6f5dfd8ff2f5cd56b08b10e33befb16c44bb79700e): Bind for>
Jun 13 08:22:59 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:59.916291062Z" level=info msg="ignoring event" container=830dd3e1f0f166cd196e6ba7ce968331c9b54a78418cfe94411ffd29b42a2da2 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:22:59 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:22:59.918733458Z" level=info msg="ignoring event" container=cf64e743992bbf2f2d7b1c850f20aa12651a0183cb73b5927e2ec00118b67152 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:23:17 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:17.827418142Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:23:17 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:17.827450325Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.047386493Z" level=info msg="ignoring event" container=ae395c2538e700fd924bbec6f2b4a5b57d7e2ea6d5245a849500cef2c0ca4e60 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.321630729Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.321746414Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.329549269Z" level=warning msg="Failed to allocate and map port 9001-9001: Bind for 0.0.0.0:9001 failed: port is already allocated"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.382935265Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.383236299Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.463762674Z" level=error msg="dad821d5302241f31cd491d688076ae6e4f1d5e464eba12f0da280a89f9db41f cleanup: failed to delete container from containerd: no such container"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.463820543Z" level=error msg="Handler for POST /v1.41/containers/dad821d5302241f31cd491d688076ae6e4f1d5e464eba12f0da280a89f9db41f/start returned error: driver failed programming external connectivity on endpoint mapp8-customer-int (c32484aaa31d246b8f460f19dc7ac8361701617569a113fb1bddaf4ac00723ce): Bind for>
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.487660545Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:23:19 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:23:19.487702291Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:52:09 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:09.862833838Z" level=info msg="ignoring event" container=37b53379da4cf48ff0242a73682a0e183b2747a91fb89afd6cab327a79efe212 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:52:09 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:09.865856024Z" level=info msg="ignoring event" container=6199c36ad2b2227d19947abfc87e45b80b3f3a81a7f055a27223691331cab279 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:52:21 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:21.967320843Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:52:21 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:21.967390358Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.190909775Z" level=info msg="ignoring event" container=9ccb3f701316409756de0bf0fd02a04ef8fd4216a0a41f4de720014c8c00cca7 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.445491986Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.445927880Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.533028998Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.533068877Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.537043683Z" level=warning msg="Failed to allocate and map port 9001-9001: Bind for 0.0.0.0:9001 failed: port is already allocated"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.622866735Z" level=error msg="064b2123bee332effd04dbd3bcffcfb8f11809ca5ab8a64a49bf51226ca7086c cleanup: failed to delete container from containerd: no such container"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.622924463Z" level=error msg="Handler for POST /v1.41/containers/064b2123bee332effd04dbd3bcffcfb8f11809ca5ab8a64a49bf51226ca7086c/start returned error: driver failed programming external connectivity on endpoint mapp8-customer-int (9f1d38c06efb5b80aeda25a48e0e7af7f9e36c79fd87b7a055551c8fa30fae20): Bind for>
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.633530605Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:52:23 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:52:23.633561674Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:54:47 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:54:47.829285003Z" level=info msg="ignoring event" container=9a5415028dc593f9fe4523fbeefe2531b35dff84f87b21c55f58bd00b911f910 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:54:47 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:54:47.834727552Z" level=info msg="ignoring event" container=7740f98f1129a3f5876cf681ccc0c01020a0b2bef6e4ff617409e2f90d7b0aea module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:56:54 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:54.728411446Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:56:54 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:54.728448756Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:56:55 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:55.897911938Z" level=info msg="ignoring event" container=0734bba3e7ee3374de333efaac9f2d2db1dca85368240b75ccd64ffbf1400ea9 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.177063023Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.177301235Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.260179273Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.260223950Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.284922803Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.284958172Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.305397156Z" level=warning msg="Failed to allocate and map port 9001-9001: Bind for 0.0.0.0:9001 failed: port is already allocated"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.399247702Z" level=error msg="b7eecf49152619fa109f575cd1058e0a6f6f9d80389349ab6c80c9d931570b1d cleanup: failed to delete container from containerd: no such container"
Jun 13 08:56:56 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:56:56.399293703Z" level=error msg="Handler for POST /v1.41/containers/b7eecf49152619fa109f575cd1058e0a6f6f9d80389349ab6c80c9d931570b1d/start returned error: driver failed programming external connectivity on endpoint mapp8-customer-int (fe4112364528b0e7d192c793929c579e8a81af715118c8f83ad7e65e7397f3be): Bind for>
Jun 13 08:59:57 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:59:57.527631039Z" level=info msg="ignoring event" container=90e2599350b1e86088807cf3919b297982655bb8cd5f09bec2d39535a35e4fdc module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 08:59:57 ip-10-0-69-193 dockerd[622]: time="2022-06-13T08:59:57.531088853Z" level=info msg="ignoring event" container=b9e863ae8291aeaa2d6ac7bcad1f6bb4bee35f0ccddfd88e31109ade1dc8c18d module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 09:00:21 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:21.007435346Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 09:00:21 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:21.007470812Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.308127921Z" level=info msg="ignoring event" container=36ebd6dc2a75e97526da0b5d9e638306c4a247d1e1bdb1a0f16df7af260959c9 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.609096106Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.609134479Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.616347863Z" level=warning msg="Failed to allocate and map port 9001-9001: Bind for 0.0.0.0:9001 failed: port is already allocated"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.700350206Z" level=error msg="02f56d9557f39c9a9851a052d3e3423192b03e96c72ca95794eff78933390533 cleanup: failed to delete container from containerd: no such container"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.700662555Z" level=error msg="Handler for POST /v1.41/containers/02f56d9557f39c9a9851a052d3e3423192b03e96c72ca95794eff78933390533/start returned error: driver failed programming external connectivity on endpoint mapp8-customer-int (72fc08854cd278e63cd3234e7fb03c08cb045efdcfb9e42075a1250d893645d5): Bind for>
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.728698048Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.728742840Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.788951270Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 09:00:22 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:00:22.788996392Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 09:32:46 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:32:46.956730677Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: [nameserver 8.8.8.8 nameserver 8.8.4.4]"
Jun 13 09:32:46 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:32:46.956771453Z" level=info msg="IPv6 enabled; Adding default IPv6 external servers: [nameserver 2001:4860:4860::8888 nameserver 2001:4860:4860::8844]"
Jun 13 09:32:46 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:32:46.961616258Z" level=warning msg="Failed to allocate and map port 8055-8055: Bind for 0.0.0.0:8055 failed: port is already allocated"
Jun 13 09:32:47 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:32:47.077575092Z" level=error msg="0bd839cbe4ebc01328a9ec8368395eadda7e72ffeffbfed42540a51dea68feca cleanup: failed to delete container from containerd: no such container"
Jun 13 09:32:47 ip-10-0-69-193 dockerd[622]: time="2022-06-13T09:32:47.077671169Z" level=error msg="Handler for POST /v1.41/containers/0bd839cbe4ebc01328a9ec8368395eadda7e72ffeffbfed42540a51dea68feca/start returned error: driver failed programming external connectivity on endpoint cms-int-contoso (b60cfae8405d9213bd1cbc583d46fbf9f7cbcbeafd2a1d4b33fa3d6162d00267): Bind for>
Status changed to 'Confirmed' because the bug affects multiple users.