[Integration tests] timeout expired waiting for neutron agents are up in some DVR tests

[Georgy Dyuldin]

Exception:
TimeoutExpired: Timeout of 600 seconds expired waiting for nodes [u'node-6.test.domain.local'] neutron agents are up

Env: Neutron(VxLAN,L2_POP,DVR)

Trace:

self = <mos_tests.neutron.python_tests.test_dvr.TestDVR object at 0x7f1235406890>
devops_env = <mos_tests.environment.devops_client.EnvProxy object at 0x7f12369dd3d0>

    @pytest.mark.testrail_id('542764')
    def test_connectivity_after_reset_compute(self, devops_env):
        """Check North-South connectivity with floatingIP after reset compute

            Scenario:
                1. Create net01, subnet net01__subnet for it
                2. Create router01 with external network and
                    router type Distributed
                3. Add interfaces to the router01 with net01__subnet
                4. Boot vm_1 in the net01
                5. Associate floating IP
                6. Go to the vm_1 with ssh and floating IP
                7. Reset compute where vm resides and wait when it's starting
                8. Go to the vm_1 with ssh and floating IP
                9. Ping 8.8.8.8
            """
        self._prepare_openstack_env()

        with self.os_conn.ssh_to_instance(self.env, self.server,
                                          self.instance_keypair) as remote:
            remote.check_call('uname -a')

        # reset compute
        compute_hostname = getattr(self.server, 'OS-EXT-SRV-ATTR:host')
> self.reset_computes([compute_hostname], devops_env)

mos_tests/neutron/python_tests/test_dvr.py:221:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <mos_tests.neutron.python_tests.test_dvr.TestDVR object at 0x7f1235406890>
hostnames = ['node-6.test.domain.local']
devops_env = <mos_tests.environment.devops_client.EnvProxy object at 0x7f12369dd3d0>

    def reset_computes(self, hostnames, devops_env):

        logger.info('Resetting computes {}'.format(hostnames))
        for hostname in hostnames:
            node = self.env.find_node_by_fqdn(hostname)
            devops_node = devops_env.get_node_by_fuel_node(node)
            devops_node.reset()

        def get_agents_on_hosts():
            agents = self.os_conn.neutron.list_agents()['agents']
            hosts_agents = [x for x in agents if x['host'] in hostnames]
            for agent in hosts_agents:
                agent['updated'] = datetime.strptime(
                    agent['heartbeat_timestamp'], "%Y-%m-%d %H:%M:%S")
            return hosts_agents

        last_updated = max(x['updated'] for x in get_agents_on_hosts())

        def is_neutron_agents_alive():
            computes_agents = get_agents_on_hosts()
            fresh_checked = [x for x in computes_agents
                             if x['updated'] > last_updated]
            alive = [x for x in fresh_checked if x['alive']]
            for agent in fresh_checked:
                state = ['is NOT', 'is'][int(agent['alive'])]
                logger.debug('{agent_type} on {host} {state} alive'.format(
                             state=state, **agent))
            return len(computes_agents) == len(alive)

        def is_nova_hypervisors_alive():
            hypervisors = [x for x in self.os_conn.nova.hypervisors.list()
                           if x.hypervisor_hostname in hostnames]
            for hypervisor in hypervisors:
                logger.debug('hypervisor on {0.hypervisor_hostname} is '
                             '{0.state}'.format(hypervisor))
            return all(x.state == 'up' for x in hypervisors)

        wait(is_neutron_agents_alive, timeout_seconds=10 * 60,
             sleep_seconds=10,
> waiting_for="nodes {0} neutron agents are up".format(hostnames))
E TimeoutExpired: Timeout of 600 seconds expired waiting for nodes [u'node-6.test.domain.local'] neutron agents are up

mos_tests/neutron/python_tests/test_dvr.py:84: TimeoutExpired

[Georgy Dyuldin (g-dyuldin)]

Commit to fix it:

https://review.gerrithub.io/#/c/277066/