## Manual Reproducer # 2 x amd64 16.04 machines juju-1.25 switch manual juju-1.25 bootstrap juju-1.25 add-machine ssh:ubuntu@10.245.168.15 juju-1.25 deploy ubuntu ubuntu-metal --to 0 juju-1.25 add-unit ubuntu-metal --to 1 juju-1.25 deploy ubuntu ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:0 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 juju-1.25 add-unit ubuntu-container --to lxc:1 ## Juju stat tabular [Services] NAME STATUS EXPOSED CHARM ubuntu-container unknown false cs:xenial/ubuntu-0 ubuntu-metal unknown false cs:xenial/ubuntu-0 [Units] ID WORKLOAD-STATE AGENT-STATE VERSION MACHINE PORTS PUBLIC-ADDRESS MESSAGE ubuntu-container/0 unknown idle 1.25.6 0/lxc/0 10.0.3.107 ubuntu-container/1 unknown idle 1.25.6 0/lxc/1 10.0.3.216 ubuntu-container/2 unknown idle 1.25.6 0/lxc/2 10.0.3.26 ubuntu-container/3 unknown idle 1.25.6 0/lxc/3 10.0.3.109 ubuntu-container/4 unknown idle 1.25.6 0/lxc/4 10.0.3.149 ubuntu-container/5 unknown idle 1.25.6 0/lxc/5 10.0.3.206 ubuntu-container/6 unknown idle 1.25.6 0/lxc/6 10.0.3.94 ubuntu-container/7 unknown idle 1.25.6 0/lxc/7 10.0.3.19 ubuntu-container/8 unknown idle 1.25.6 0/lxc/8 10.0.3.174 ubuntu-container/9 unknown idle 1.25.6 0/lxc/9 10.0.3.213 ubuntu-container/10 unknown idle 1.25.6 1/lxc/0 10.0.3.35 ubuntu-container/11 unknown idle 1.25.6 1/lxc/1 10.0.3.188 ubuntu-container/12 unknown idle 1.25.6 1/lxc/2 10.0.3.104 ubuntu-container/13 unknown idle 1.25.6 1/lxc/3 10.0.3.185 ubuntu-container/14 unknown idle 1.25.6 1/lxc/4 10.0.3.209 ubuntu-container/15 unknown idle 1.25.6 1/lxc/5 10.0.3.230 ubuntu-container/16 unknown idle 1.25.6 1/lxc/6 10.0.3.161 ubuntu-container/17 unknown idle 1.25.6 1/lxc/7 10.0.3.40 ubuntu-container/18 unknown idle 1.25.6 1/lxc/8 10.0.3.146 ubuntu-container/19 unknown idle 1.25.6 1/lxc/9 10.0.3.196 ubuntu-metal/0 unknown idle 1.25.6 0 10.245.168.17 ubuntu-metal/1 unknown idle 1.25.6 1 10.245.168.15 [Machines] ID STATE VERSION DNS INS-ID SERIES HARDWARE 0 started 1.25.6 10.245.168.17 manual: xenial arch=amd64 cpu-cores=4 mem=48287M 1 started 1.25.6 10.245.168.15 manual:10.245.168.15 xenial arch=amd64 cpu-cores=4 mem=48287M ## Juju stat yaml environment: manual machines: "0": agent-state: started agent-version: 1.25.6 dns-name: 10.245.168.17 instance-id: 'manual:' series: xenial containers: 0/lxc/0: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.107 instance-id: juju-machine-0-lxc-0 series: xenial hardware: arch=amd64 0/lxc/1: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.216 instance-id: juju-machine-0-lxc-1 series: xenial hardware: arch=amd64 0/lxc/2: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.26 instance-id: juju-machine-0-lxc-2 series: xenial hardware: arch=amd64 0/lxc/3: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.109 instance-id: juju-machine-0-lxc-3 series: xenial hardware: arch=amd64 0/lxc/4: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.149 instance-id: juju-machine-0-lxc-4 series: xenial hardware: arch=amd64 0/lxc/5: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.206 instance-id: juju-machine-0-lxc-5 series: xenial hardware: arch=amd64 0/lxc/6: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.94 instance-id: juju-machine-0-lxc-6 series: xenial hardware: arch=amd64 0/lxc/7: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.19 instance-id: juju-machine-0-lxc-7 series: xenial hardware: arch=amd64 0/lxc/8: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.174 instance-id: juju-machine-0-lxc-8 series: xenial hardware: arch=amd64 0/lxc/9: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.213 instance-id: juju-machine-0-lxc-9 series: xenial hardware: arch=amd64 hardware: arch=amd64 cpu-cores=4 mem=48287M state-server-member-status: has-vote "1": agent-state: started agent-version: 1.25.6 dns-name: 10.245.168.15 instance-id: manual:10.245.168.15 series: xenial containers: 1/lxc/0: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.35 instance-id: juju-machine-1-lxc-0 series: xenial hardware: arch=amd64 1/lxc/1: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.188 instance-id: juju-machine-1-lxc-1 series: xenial hardware: arch=amd64 1/lxc/2: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.104 instance-id: juju-machine-1-lxc-2 series: xenial hardware: arch=amd64 1/lxc/3: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.185 instance-id: juju-machine-1-lxc-3 series: xenial hardware: arch=amd64 1/lxc/4: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.209 instance-id: juju-machine-1-lxc-4 series: xenial hardware: arch=amd64 1/lxc/5: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.230 instance-id: juju-machine-1-lxc-5 series: xenial hardware: arch=amd64 1/lxc/6: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.161 instance-id: juju-machine-1-lxc-6 series: xenial hardware: arch=amd64 1/lxc/7: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.40 instance-id: juju-machine-1-lxc-7 series: xenial hardware: arch=amd64 1/lxc/8: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.146 instance-id: juju-machine-1-lxc-8 series: xenial hardware: arch=amd64 1/lxc/9: agent-state: started agent-version: 1.25.6 dns-name: 10.0.3.196 instance-id: juju-machine-1-lxc-9 series: xenial hardware: arch=amd64 hardware: arch=amd64 cpu-cores=4 mem=48287M services: ubuntu-container: charm: cs:xenial/ubuntu-0 exposed: false service-status: current: unknown since: 18 Aug 2016 08:15:43-05:00 units: ubuntu-container/0: workload-status: current: unknown since: 18 Aug 2016 08:15:43-05:00 agent-status: current: idle since: 18 Aug 2016 09:25:46-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/0 public-address: 10.0.3.107 ubuntu-container/1: workload-status: current: unknown since: 18 Aug 2016 08:16:44-05:00 agent-status: current: idle since: 18 Aug 2016 09:21:47-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/1 public-address: 10.0.3.216 ubuntu-container/2: workload-status: current: unknown since: 18 Aug 2016 08:16:48-05:00 agent-status: current: idle since: 18 Aug 2016 09:21:51-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/2 public-address: 10.0.3.26 ubuntu-container/3: workload-status: current: unknown since: 18 Aug 2016 08:16:48-05:00 agent-status: current: idle since: 18 Aug 2016 09:21:51-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/3 public-address: 10.0.3.109 ubuntu-container/4: workload-status: current: unknown since: 18 Aug 2016 08:17:02-05:00 agent-status: current: idle since: 18 Aug 2016 09:22:05-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/4 public-address: 10.0.3.149 ubuntu-container/5: workload-status: current: unknown since: 18 Aug 2016 08:17:56-05:00 agent-status: current: idle since: 18 Aug 2016 09:22:59-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/5 public-address: 10.0.3.206 ubuntu-container/6: workload-status: current: unknown since: 18 Aug 2016 08:18:00-05:00 agent-status: current: idle since: 18 Aug 2016 09:23:03-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/6 public-address: 10.0.3.94 ubuntu-container/7: workload-status: current: unknown since: 18 Aug 2016 08:18:00-05:00 agent-status: current: idle since: 18 Aug 2016 09:23:03-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/7 public-address: 10.0.3.19 ubuntu-container/8: workload-status: current: unknown since: 18 Aug 2016 08:18:05-05:00 agent-status: current: idle since: 18 Aug 2016 09:23:07-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/8 public-address: 10.0.3.174 ubuntu-container/9: workload-status: current: unknown since: 18 Aug 2016 08:16:49-05:00 agent-status: current: idle since: 18 Aug 2016 09:21:52-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 0/lxc/9 public-address: 10.0.3.213 ubuntu-container/10: workload-status: current: unknown since: 18 Aug 2016 09:18:16-05:00 agent-status: current: idle since: 18 Aug 2016 09:23:33-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/0 public-address: 10.0.3.35 ubuntu-container/11: workload-status: current: unknown since: 18 Aug 2016 09:19:11-05:00 agent-status: current: idle since: 18 Aug 2016 09:24:14-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/1 public-address: 10.0.3.188 ubuntu-container/12: workload-status: current: unknown since: 18 Aug 2016 09:19:12-05:00 agent-status: current: idle since: 18 Aug 2016 09:24:15-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/2 public-address: 10.0.3.104 ubuntu-container/13: workload-status: current: unknown since: 18 Aug 2016 09:19:13-05:00 agent-status: current: idle since: 18 Aug 2016 09:24:16-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/3 public-address: 10.0.3.185 ubuntu-container/14: workload-status: current: unknown since: 18 Aug 2016 09:19:28-05:00 agent-status: current: idle since: 18 Aug 2016 09:24:32-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/4 public-address: 10.0.3.209 ubuntu-container/15: workload-status: current: unknown since: 18 Aug 2016 09:19:40-05:00 agent-status: current: idle since: 18 Aug 2016 09:24:50-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/5 public-address: 10.0.3.230 ubuntu-container/16: workload-status: current: unknown since: 18 Aug 2016 09:18:05-05:00 agent-status: current: idle since: 18 Aug 2016 09:23:08-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/6 public-address: 10.0.3.161 ubuntu-container/17: workload-status: current: unknown since: 18 Aug 2016 09:20:19-05:00 agent-status: current: idle since: 18 Aug 2016 09:25:22-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/7 public-address: 10.0.3.40 ubuntu-container/18: workload-status: current: unknown since: 18 Aug 2016 09:20:22-05:00 agent-status: current: idle since: 18 Aug 2016 09:25:24-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/8 public-address: 10.0.3.146 ubuntu-container/19: workload-status: current: unknown since: 18 Aug 2016 09:20:21-05:00 agent-status: current: idle since: 18 Aug 2016 09:25:24-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: 1/lxc/9 public-address: 10.0.3.196 ubuntu-metal: charm: cs:xenial/ubuntu-0 exposed: false service-status: current: unknown since: 18 Aug 2016 08:12:30-05:00 units: ubuntu-metal/0: workload-status: current: unknown since: 18 Aug 2016 08:12:30-05:00 agent-status: current: idle since: 18 Aug 2016 09:22:57-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: "0" public-address: 10.245.168.17 ubuntu-metal/1: workload-status: current: unknown since: 18 Aug 2016 09:17:09-05:00 agent-status: current: idle since: 18 Aug 2016 09:22:12-05:00 version: 1.25.6 agent-state: started agent-version: 1.25.6 machine: "1" public-address: 10.245.168.15 ## Check connectivity to everything from laptop # All containers are unreachable. rbeisner@rby:~⟫ for i in $(juju-1.25 status --format yaml | grep public-address | awk '{ print $2 }'); do echo $i; nc -vzw 1 $i 22; done 10.0.3.107 nc: connect to 10.0.3.107 port 22 (tcp) timed out: Operation now in progress 10.0.3.216 nc: connect to 10.0.3.216 port 22 (tcp) timed out: Operation now in progress 10.0.3.26 nc: connect to 10.0.3.26 port 22 (tcp) timed out: Operation now in progress 10.0.3.109 nc: connect to 10.0.3.109 port 22 (tcp) timed out: Operation now in progress 10.0.3.149 nc: connect to 10.0.3.149 port 22 (tcp) timed out: Operation now in progress 10.0.3.206 nc: connect to 10.0.3.206 port 22 (tcp) timed out: Operation now in progress 10.0.3.94 nc: connect to 10.0.3.94 port 22 (tcp) timed out: Operation now in progress 10.0.3.19 nc: connect to 10.0.3.19 port 22 (tcp) timed out: Operation now in progress 10.0.3.174 nc: connect to 10.0.3.174 port 22 (tcp) timed out: Operation now in progress 10.0.3.213 nc: connect to 10.0.3.213 port 22 (tcp) timed out: Operation now in progress 10.0.3.35 nc: connect to 10.0.3.35 port 22 (tcp) timed out: Operation now in progress 10.0.3.188 nc: connect to 10.0.3.188 port 22 (tcp) timed out: Operation now in progress 10.0.3.104 nc: connect to 10.0.3.104 port 22 (tcp) timed out: Operation now in progress 10.0.3.185 nc: connect to 10.0.3.185 port 22 (tcp) timed out: Operation now in progress 10.0.3.209 nc: connect to 10.0.3.209 port 22 (tcp) timed out: Operation now in progress 10.0.3.230 nc: connect to 10.0.3.230 port 22 (tcp) timed out: Operation now in progress 10.0.3.161 nc: connect to 10.0.3.161 port 22 (tcp) timed out: Operation now in progress 10.0.3.40 nc: connect to 10.0.3.40 port 22 (tcp) timed out: Operation now in progress 10.0.3.146 nc: connect to 10.0.3.146 port 22 (tcp) timed out: Operation now in progress 10.0.3.196 nc: connect to 10.0.3.196 port 22 (tcp) timed out: Operation now in progress 10.245.168.17 Connection to 10.245.168.17 22 port [tcp/ssh] succeeded! 10.245.168.15 Connection to 10.245.168.15 22 port [tcp/ssh] succeeded! rbeisner@rby:~/snap/ci-dice⟫ ## Check connectivity to everything from machine 0 # Only containers on machine 0 are reachable. export iplist="10.0.3.107 10.0.3.216 10.0.3.26 10.0.3.109 10.0.3.149 10.0.3.206 10.0.3.94 10.0.3.19 10.0.3.174 10.0.3.213 10.0.3.35 10.0.3.188 10.0.3.104 10.0.3.185 10.0.3.209 10.0.3.230 10.0.3.161 10.0.3.40 10.0.3.146 10.0.3.196 10.245.168.17 10.245.168.15" ubuntu@reflecting-attraction:~⟫ for i in $iplist; do echo $i; nc -vzw 1 $i 22; done 10.0.3.107 Connection to 10.0.3.107 22 port [tcp/ssh] succeeded! 10.0.3.216 Connection to 10.0.3.216 22 port [tcp/ssh] succeeded! 10.0.3.26 Connection to 10.0.3.26 22 port [tcp/ssh] succeeded! 10.0.3.109 Connection to 10.0.3.109 22 port [tcp/ssh] succeeded! 10.0.3.149 Connection to 10.0.3.149 22 port [tcp/ssh] succeeded! 10.0.3.206 Connection to 10.0.3.206 22 port [tcp/ssh] succeeded! 10.0.3.94 Connection to 10.0.3.94 22 port [tcp/ssh] succeeded! 10.0.3.19 Connection to 10.0.3.19 22 port [tcp/ssh] succeeded! 10.0.3.174 Connection to 10.0.3.174 22 port [tcp/ssh] succeeded! 10.0.3.213 Connection to 10.0.3.213 22 port [tcp/ssh] succeeded! 10.0.3.35 nc: connect to 10.0.3.35 port 22 (tcp) timed out: Operation now in progress 10.0.3.188 nc: connect to 10.0.3.188 port 22 (tcp) timed out: Operation now in progress 10.0.3.104 nc: connect to 10.0.3.104 port 22 (tcp) timed out: Operation now in progress 10.0.3.185 nc: connect to 10.0.3.185 port 22 (tcp) timed out: Operation now in progress 10.0.3.209 nc: connect to 10.0.3.209 port 22 (tcp) timed out: Operation now in progress 10.0.3.230 nc: connect to 10.0.3.230 port 22 (tcp) timed out: Operation now in progress 10.0.3.161 nc: connect to 10.0.3.161 port 22 (tcp) timed out: Operation now in progress 10.0.3.40 nc: connect to 10.0.3.40 port 22 (tcp) timed out: Operation now in progress 10.0.3.146 nc: connect to 10.0.3.146 port 22 (tcp) timed out: Operation now in progress 10.0.3.196 nc: connect to 10.0.3.196 port 22 (tcp) timed out: Operation now in progress 10.245.168.17 Connection to 10.245.168.17 22 port [tcp/ssh] succeeded! 10.245.168.15 Connection to 10.245.168.15 22 port [tcp/ssh] succeeded! ## Check connectivity to everything from machine 1 # Only containers on machine 1 are reachable. export iplist="10.0.3.107 10.0.3.216 10.0.3.26 10.0.3.109 10.0.3.149 10.0.3.206 10.0.3.94 10.0.3.19 10.0.3.174 10.0.3.213 10.0.3.35 10.0.3.188 10.0.3.104 10.0.3.185 10.0.3.209 10.0.3.230 10.0.3.161 10.0.3.40 10.0.3.146 10.0.3.196 10.245.168.17 10.245.168.15" ubuntu@shady-wave:~$ for i in $iplist; do echo $i; nc -vzw 1 $i 22; done 10.0.3.107 nc: connect to 10.0.3.107 port 22 (tcp) timed out: Operation now in progress 10.0.3.216 nc: connect to 10.0.3.216 port 22 (tcp) timed out: Operation now in progress 10.0.3.26 nc: connect to 10.0.3.26 port 22 (tcp) timed out: Operation now in progress 10.0.3.109 nc: connect to 10.0.3.109 port 22 (tcp) timed out: Operation now in progress 10.0.3.149 nc: connect to 10.0.3.149 port 22 (tcp) timed out: Operation now in progress 10.0.3.206 nc: connect to 10.0.3.206 port 22 (tcp) timed out: Operation now in progress 10.0.3.94 nc: connect to 10.0.3.94 port 22 (tcp) timed out: Operation now in progress 10.0.3.19 nc: connect to 10.0.3.19 port 22 (tcp) timed out: Operation now in progress 10.0.3.174 nc: connect to 10.0.3.174 port 22 (tcp) timed out: Operation now in progress 10.0.3.213 nc: connect to 10.0.3.213 port 22 (tcp) timed out: Operation now in progress 10.0.3.35 Connection to 10.0.3.35 22 port [tcp/ssh] succeeded! 10.0.3.188 Connection to 10.0.3.188 22 port [tcp/ssh] succeeded! 10.0.3.104 Connection to 10.0.3.104 22 port [tcp/ssh] succeeded! 10.0.3.185 Connection to 10.0.3.185 22 port [tcp/ssh] succeeded! 10.0.3.209 Connection to 10.0.3.209 22 port [tcp/ssh] succeeded! 10.0.3.230 Connection to 10.0.3.230 22 port [tcp/ssh] succeeded! 10.0.3.161 Connection to 10.0.3.161 22 port [tcp/ssh] succeeded! 10.0.3.40 Connection to 10.0.3.40 22 port [tcp/ssh] succeeded! 10.0.3.146 Connection to 10.0.3.146 22 port [tcp/ssh] succeeded! 10.0.3.196 Connection to 10.0.3.196 22 port [tcp/ssh] succeeded! 10.245.168.17 Connection to 10.245.168.17 22 port [tcp/ssh] succeeded! 10.245.168.15 Connection to 10.245.168.15 22 port [tcp/ssh] succeeded!