It looks like this is a problem with grpcio-1.16.0+. If I add a version cap to keep it below 1.16.0 then everything works as expected.
Also, since I forgot to include the traceback in the original report, here it is:
2018-12-11 00:21:10.259089 | ubuntu-xenial | Captured traceback: 2018-12-11 00:21:10.259155 | ubuntu-xenial | ~~~~~~~~~~~~~~~~~~~ 2018-12-11 00:21:10.259260 | ubuntu-xenial | b'Traceback (most recent call last):' 2018-12-11 00:21:10.259565 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/.tox/py35-etcd3/lib/python3.5/site-packages/etcd3/client.py", line 46, in handler' 2018-12-11 00:21:10.259666 | ubuntu-xenial | b' return f(*args, **kwargs)' 2018-12-11 00:21:10.259943 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/.tox/py35-etcd3/lib/python3.5/site-packages/etcd3/client.py", line 679, in lease' 2018-12-11 00:21:10.260044 | ubuntu-xenial | b' metadata=self.metadata' 2018-12-11 00:21:10.260346 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/.tox/py35-etcd3/lib/python3.5/site-packages/grpc/_channel.py", line 547, in __call__' 2018-12-11 00:21:10.260518 | ubuntu-xenial | b' return _end_unary_response_blocking(state, call, False, None)' 2018-12-11 00:21:10.260855 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/.tox/py35-etcd3/lib/python3.5/site-packages/grpc/_channel.py", line 466, in _end_unary_response_blocking' 2018-12-11 00:21:10.260984 | ubuntu-xenial | b' raise _Rendezvous(state, None, None, deadline)' 2018-12-11 00:21:10.261137 | ubuntu-xenial | b'grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:' 2018-12-11 00:21:10.261248 | ubuntu-xenial | b'\tstatus = StatusCode.DEADLINE_EXCEEDED' 2018-12-11 00:21:10.261346 | ubuntu-xenial | b'\tdetails = "Deadline Exceeded"' 2018-12-11 00:21:10.262327 | ubuntu-xenial | b'\tdebug_error_string = "{"created":"@1544487511.159561332","description":"Failed to create subchannel","file":"src/core/ext/filters/client_channel/client_channel.cc","file_line":2706,"referenced_errors":[{"created":"@1544487511.159555724","description":"Pick Cancelled","file":"src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc","file_line":221,"referenced_errors":[{"created":"@1544487511.159549203","description":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":69,"grpc_status":4}]}]}"' 2018-12-11 00:21:10.262378 | ubuntu-xenial | b'>' 2018-12-11 00:21:10.262420 | ubuntu-xenial | b'' 2018-12-11 00:21:10.262571 | ubuntu-xenial | b'During handling of the above exception, another exception occurred:' 2018-12-11 00:21:10.262617 | ubuntu-xenial | b'' 2018-12-11 00:21:10.262719 | ubuntu-xenial | b'Traceback (most recent call last):' 2018-12-11 00:21:10.262960 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/tooz/tests/__init__.py", line 36, in skip_if_not_implemented' 2018-12-11 00:21:10.263058 | ubuntu-xenial | b' return func(*args, **kwargs)' 2018-12-11 00:21:10.263296 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/tooz/tests/test_coordination.py", line 355, in test_timeout' 2018-12-11 00:21:10.263380 | ubuntu-xenial | b' self._coord.start()' 2018-12-11 00:21:10.263606 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/tooz/coordination.py", line 690, in start' 2018-12-11 00:21:10.263762 | ubuntu-xenial | b' super(CoordinationDriverWithExecutor, self).start(start_heart)' 2018-12-11 00:21:10.263972 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/tooz/coordination.py", line 426, in start' 2018-12-11 00:21:10.264052 | ubuntu-xenial | b' self._start()' 2018-12-11 00:21:10.264257 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/tooz/drivers/etcd3.py", line 159, in _start' 2018-12-11 00:21:10.264422 | ubuntu-xenial | b' self._membership_lease = self.client.lease(self.membership_timeout)' 2018-12-11 00:21:10.264708 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/.tox/py35-etcd3/lib/python3.5/site-packages/etcd3/client.py", line 48, in handler' 2018-12-11 00:21:10.264817 | ubuntu-xenial | b' _translate_exception(exc)' 2018-12-11 00:21:10.265132 | ubuntu-xenial | b' File "/home/zuul/src/git.openstack.org/openstack/tooz/.tox/py35-etcd3/lib/python3.5/site-packages/etcd3/client.py", line 32, in _translate_exception' 2018-12-11 00:21:10.265209 | ubuntu-xenial | b' raise exception' 2018-12-11 00:21:10.265326 | ubuntu-xenial | b'etcd3.exceptions.ConnectionTimeoutError' 2018-12-11 00:21:10.265369 | ubuntu-xenial | b''
It looks like this is a problem with grpcio-1.16.0+. If I add a version cap to keep it below 1.16.0 then everything works as expected.
Also, since I forgot to include the traceback in the original report, here it is:
2018-12-11 00:21:10.259089 | ubuntu-xenial | Captured traceback: zuul/src/ git.openstack. org/openstack/ tooz/.tox/ py35-etcd3/ lib/python3. 5/site- packages/ etcd3/client. py", line 46, in handler' zuul/src/ git.openstack. org/openstack/ tooz/.tox/ py35-etcd3/ lib/python3. 5/site- packages/ etcd3/client. py", line 679, in lease' self.metadata' zuul/src/ git.openstack. org/openstack/ tooz/.tox/ py35-etcd3/ lib/python3. 5/site- packages/ grpc/_channel. py", line 547, in __call__' response_ blocking( state, call, False, None)' zuul/src/ git.openstack. org/openstack/ tooz/.tox/ py35-etcd3/ lib/python3. 5/site- packages/ grpc/_channel. py", line 466, in _end_unary_ response_ blocking' _channel. _Rendezvous: <_Rendezvous of RPC that terminated with:' DEADLINE_ EXCEEDED' error_string = "{"created" :"@1544487511. 159561332" ,"description" :"Failed to create subchannel" ,"file" :"src/core/ ext/filters/ client_ channel/ client_ channel. cc","file_ line":2706, "referenced_ errors" :[{"created" :"@1544487511. 159555724" ,"description" :"Pick Cancelled" ,"file" :"src/core/ ext/filters/ client_ channel/ lb_policy/ pick_first/ pick_first. cc","file_ line":221, "referenced_ errors" :[{"created" :"@1544487511. 159549203" ,"description" :"Deadline Exceeded" ,"file" :"src/core/ ext/filters/ deadline/ deadline_ filter. cc","file_ line":69, "grpc_status" :4}]}]} "' zuul/src/ git.openstack. org/openstack/ tooz/tooz/ tests/_ _init__ .py", line 36, in skip_if_ not_implemented ' zuul/src/ git.openstack. org/openstack/ tooz/tooz/ tests/test_ coordination. py", line 355, in test_timeout' start() ' zuul/src/ git.openstack. org/openstack/ tooz/tooz/ coordination. py", line 690, in start' ionDriverWithEx ecutor, self).start( start_heart) ' zuul/src/ git.openstack. org/openstack/ tooz/tooz/ coordination. py", line 426, in start' zuul/src/ git.openstack. org/openstack/ tooz/tooz/ drivers/ etcd3.py" , line 159, in _start' p_lease = self.client. lease(self. membership_ timeout) ' zuul/src/ git.openstack. org/openstack/ tooz/.tox/ py35-etcd3/ lib/python3. 5/site- packages/ etcd3/client. py", line 48, in handler' exception( exc)' zuul/src/ git.openstack. org/openstack/ tooz/.tox/ py35-etcd3/ lib/python3. 5/site- packages/ etcd3/client. py", line 32, in _translate_ exception' exceptions. ConnectionTimeo utError'
2018-12-11 00:21:10.259155 | ubuntu-xenial | ~~~~~~~~~~~~~~~~~~~
2018-12-11 00:21:10.259260 | ubuntu-xenial | b'Traceback (most recent call last):'
2018-12-11 00:21:10.259565 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.259666 | ubuntu-xenial | b' return f(*args, **kwargs)'
2018-12-11 00:21:10.259943 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.260044 | ubuntu-xenial | b' metadata=
2018-12-11 00:21:10.260346 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.260518 | ubuntu-xenial | b' return _end_unary_
2018-12-11 00:21:10.260855 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.260984 | ubuntu-xenial | b' raise _Rendezvous(state, None, None, deadline)'
2018-12-11 00:21:10.261137 | ubuntu-xenial | b'grpc.
2018-12-11 00:21:10.261248 | ubuntu-xenial | b'\tstatus = StatusCode.
2018-12-11 00:21:10.261346 | ubuntu-xenial | b'\tdetails = "Deadline Exceeded"'
2018-12-11 00:21:10.262327 | ubuntu-xenial | b'\tdebug_
2018-12-11 00:21:10.262378 | ubuntu-xenial | b'>'
2018-12-11 00:21:10.262420 | ubuntu-xenial | b''
2018-12-11 00:21:10.262571 | ubuntu-xenial | b'During handling of the above exception, another exception occurred:'
2018-12-11 00:21:10.262617 | ubuntu-xenial | b''
2018-12-11 00:21:10.262719 | ubuntu-xenial | b'Traceback (most recent call last):'
2018-12-11 00:21:10.262960 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.263058 | ubuntu-xenial | b' return func(*args, **kwargs)'
2018-12-11 00:21:10.263296 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.263380 | ubuntu-xenial | b' self._coord.
2018-12-11 00:21:10.263606 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.263762 | ubuntu-xenial | b' super(Coordinat
2018-12-11 00:21:10.263972 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.264052 | ubuntu-xenial | b' self._start()'
2018-12-11 00:21:10.264257 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.264422 | ubuntu-xenial | b' self._membershi
2018-12-11 00:21:10.264708 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.264817 | ubuntu-xenial | b' _translate_
2018-12-11 00:21:10.265132 | ubuntu-xenial | b' File "/home/
2018-12-11 00:21:10.265209 | ubuntu-xenial | b' raise exception'
2018-12-11 00:21:10.265326 | ubuntu-xenial | b'etcd3.
2018-12-11 00:21:10.265369 | ubuntu-xenial | b''