[integration tests] Timeout exception during ping on some DVR tests

[Georgy Dyuldin]

Test result:

https://mirantis.testrail.com/index.php?/tests/view/5881180

Exception:

TimeoutExpired: Timeout of 180 seconds expired waiting for SSH command: `ping -c1 8.8.8.8` completed with 0 exit code

Trace:

self = <mos_tests.neutron.python_tests.test_dvr.TestDVR object at 0x7f9e5eb8bb10>
count = 40

    @pytest.mark.testrail_id('674297')
    def test_connectivity_after_ban_l3_agent_many_times(self, count=40):
        """Check North-South connectivity without floating after ban l3 agent
                many times

            Scenario:
                1. Create net1, subnet1
                2. Create DVR router router1, set gateway and add interface to net1
                3. Boot vm in net1
                4. Check that ping 8.8.8.8 available from vm
                5. Find node with snat for router1:
                    ip net | grep snat-<id_router> on each controller
                6. Ban other l3-agents
                7. Ban l3-agent on for node with snat:
                    pcs resource ban neutron-l3-agent <controller>
                8. Wait 10 seconds
                9. Clear l3-agent on for node with snat:
                    pcs resource clear neutron-l3-agent <controller>
                10. Repeat steps 7-9 `count` times
                11. Check that ping 8.8.8.8 available from vm
            """
        self._prepare_openstack_env(assign_floating_ip=False)

        controller = self.find_snat_controller(self.router_id)
        controllers = self.env.get_nodes_by_role('controller')

        # Ban all l3 agents
        with controller.ssh() as remote:
            logger.info('Ban all l3 agents, except placed on {}'.format(
                controller))
            for agent in self.os_conn.list_l3_agents():
                if agent['host'] not in [x.data['fqdn'] for x in controllers]:
                    continue
                if agent['host'] == controller.data['fqdn']:
                    continue
                remote.check_call(
                    'pcs resource ban neutron-l3-agent {host}'.format(
                        **agent))

        cmd = 'pcs resource {{action}} neutron-l3-agent {fqdn}'.format(
                **controller.data)
        with controller.ssh() as remote:
            for i in range(1, 41):
                logger.info('Ban/clear l3 agent on {node} - {i}'.format(
                    node=controller, i=i))
                remote.check_call(cmd.format(action='ban'))
                time.sleep(10)
                remote.check_call(cmd.format(action='clear'))
                time.sleep(10)

        network_checks.check_ping_from_vm(self.env, self.os_conn, self.server,
> vm_keypair=self.instance_keypair)

mos_tests/neutron/python_tests/test_dvr.py:418:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
mos_tests/functions/network_checks.py:73: in check_ping_from_vm
    vm_password)
mos_tests/functions/network_checks.py:98: in check_ping_from_vm_helper
    vm_login=vm_login, vm_password=vm_password)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

env = <mos_tests.environment.fuel_client.Environment object at 0x7f9e5ecebc10>
os_conn = <mos_tests.environment.os_actions.OpenStackActions object at 0x7f9e5ee3d050>
vm = <Server: server01>, vm_keypair = <Keypair: instancekey>
command = 'ping -c1 8.8.8.8', vm_login = 'cirros', timeout = 180
vm_password = 'cubswin:)'

    def run_on_vm(env, os_conn, vm, vm_keypair=None, command='uname',
                  vm_login="cirros", timeout=3 * 60, vm_password='cubswin:)'):
        """Execute command on vm and return dict with results

        :param vm: server to execute command on
        :param vm_keypair: keypair used during vm creating
        :param command: command to execute
        :param vm_login: username to login to vm via ssh
        :param vm_password: password to login to vm via ssh
        :param timeout: type - int or None
            - if None - execute command and return results
            - if int - wait `timeout` seconds until command exit_code will be 0
        :returns: Dictionary with `exit_code`, `stdout`, `stderr` keys.
            `Stdout` and `stderr` are list of strings
        """
        results = []

        def execute():
            with os_conn.ssh_to_instance(env, vm, vm_keypair,
                                         username=vm_login,
                                         password=vm_password) as remote:
                result = remote.execute(command)
                results.append(result)
                return result

        logger.info('Executing `{cmd}` on {vm_name}'.format(
            cmd=command,
            vm_name=vm.name))

        if timeout is None:
            execute()
        else:
            err_msg = "SSH command: `{command}` completed with 0 exit code"
            wait(lambda: execute()['exit_code'] == 0,
                 sleep_seconds=(1, 60, 5), timeout_seconds=timeout,
                 expected_exceptions=(Exception,),
> waiting_for=err_msg.format(command=command))
E TimeoutExpired: Timeout of 180 seconds expired waiting for SSH command: `ping -c1 8.8.8.8` completed with 0 exit code

mos_tests/functions/network_checks.py:62: TimeoutExpired