I just built a testcase that was able to reproduce that behaviour. It can be reproduced with the following code in `neutron/tests/functional/services/ovn_l3/test_plugin.py`

```
    def test_gateway_chassis_does_not_become_unbalanced(self):
        def print_prio_by_chassis():
            chassis = self.nb_api.get_all_chassis_gateway_bindings()
            chassis_prios = {}
            for chassis_name in chassis:
                prios = {}
                for (_, prio) in chassis[chassis_name]:
                    prios[prio] = prios.setdefault(prio, 0) + 1
                chassis_prios[chassis_name] = prios
            for chassis_name in sorted(chassis_prios):
                print("%s:" % chassis_name)
                for prio, count in sorted(chassis_prios[chassis_name].items()):
                    print("\tprio %s: %s routers" % (prio, count))

        # test with 6 gw chassis and 24 lrps
        chassis_list = []
        for i in range(0, ovn_const.MAX_GW_CHASSIS + 1):
            name = 'ovs-host%s' % i
            chassis_list.append(
                self.add_fake_chassis(
                    name, physical_nets=['physnet1'], name=name,
                    other_config={'ovn-cms-options': 'enable-chassis-as-gw'}))

        ext1 = self._create_ext_network(
            'ext1', 'vlan', 'physnet1', 1, "10.0.0.1", "10.0.0.0/24")
        gw_info = {'network_id': ext1['network']['id']}
        for i in range(0, 24):
            router = self._create_router('router-%s' % i, gw_info=gw_info)
            gw_port_id = router.get('gw_port_id')
            logical_port = 'cr-lrp-%s' % gw_port_id
            self.assertTrue(self.cr_lrp_pb_event.wait(logical_port),
                            msg='lrp %s failed to bind' % logical_port)
            self.sb_api.lsp_bind(logical_port, chassis_list[0],
                                may_exist=True).execute(check_error=True)

        print("Initial setup")
        self.l3_plugin.schedule_unhosted_gateways()
        print_prio_by_chassis()
        print()
        print()

        print("Now evicting each router once")
        for i in range(0, ovn_const.MAX_GW_CHASSIS + 1):
            c = chassis_list[i]
            del chassis_list[i]
            self.del_fake_chassis(c)
            print("host %s now gone" % i)
            self.l3_plugin.schedule_unhosted_gateways()
            print_prio_by_chassis()
            print()
            print()

            name = 'ovs-host%s' % i
            chassis_list.insert(
                i,
                self.add_fake_chassis(
                    name, physical_nets=['physnet1'], name=name,
                    other_config={'ovn-cms-options': 'enable-chassis-as-gw'}))
            print("host %s now back" % i)
            self.l3_plugin.schedule_unhosted_gateways()
            print_prio_by_chassis()
            print()
            print()

        self.assertFalse(True)
```

---

This will output the following data clearly showing our issue:

```
    Initial setup
ovs-host0:
        prio 1: 5 routers
        prio 2: 2 routers
        prio 3: 5 routers
        prio 4: 4 routers
        prio 5: 4 routers
ovs-host1:
        prio 1: 4 routers
        prio 2: 5 routers
        prio 3: 5 routers
        prio 4: 2 routers
        prio 5: 4 routers
ovs-host2:
        prio 1: 4 routers
        prio 2: 3 routers
        prio 3: 3 routers
        prio 4: 7 routers
        prio 5: 4 routers
ovs-host3:
        prio 1: 4 routers
        prio 2: 5 routers
        prio 3: 5 routers
        prio 4: 2 routers
        prio 5: 4 routers
ovs-host4:
        prio 1: 5 routers
        prio 2: 4 routers
        prio 3: 4 routers
        prio 4: 3 routers
        prio 5: 4 routers
ovs-host5:
        prio 1: 2 routers
        prio 2: 5 routers
        prio 3: 2 routers
        prio 4: 6 routers
        prio 5: 4 routers


Now evicting each router once
host 0 now gone
ovs-host1:
        prio 1: 4 routers
        prio 2: 4 routers
        prio 3: 10 routers
        prio 4: 2 routers
        prio 5: 4 routers
ovs-host2:
        prio 1: 5 routers
        prio 2: 3 routers
        prio 3: 3 routers
        prio 4: 7 routers
        prio 5: 6 routers
ovs-host3:
        prio 1: 4 routers
        prio 2: 7 routers
        prio 3: 5 routers
        prio 4: 3 routers
        prio 5: 5 routers
ovs-host4:
        prio 1: 6 routers
        prio 2: 7 routers
        prio 3: 2 routers
        prio 4: 5 routers
        prio 5: 4 routers
ovs-host5:
        prio 1: 5 routers
        prio 2: 3 routers
        prio 3: 4 routers
        prio 4: 7 routers
        prio 5: 5 routers


host 0 now back
ovs-host1:
        prio 1: 4 routers
        prio 2: 4 routers
        prio 3: 10 routers
        prio 4: 2 routers
        prio 5: 4 routers
ovs-host2:
        prio 1: 5 routers
        prio 2: 3 routers
        prio 3: 3 routers
        prio 4: 7 routers
        prio 5: 6 routers
ovs-host3:
        prio 1: 4 routers
        prio 2: 7 routers
        prio 3: 5 routers
        prio 4: 3 routers
        prio 5: 5 routers
ovs-host4:
        prio 1: 6 routers
        prio 2: 7 routers
        prio 3: 2 routers
        prio 4: 5 routers
        prio 5: 4 routers
ovs-host5:
        prio 1: 5 routers
        prio 2: 3 routers
        prio 3: 4 routers
        prio 4: 7 routers
        prio 5: 5 routers


host 1 now gone
ovs-host0:
        prio 1: 24 routers
ovs-host2:
        prio 2: 6 routers
        prio 3: 3 routers
        prio 4: 8 routers
        prio 5: 7 routers
ovs-host3:
        prio 2: 4 routers
        prio 3: 9 routers
        prio 4: 6 routers
        prio 5: 5 routers
ovs-host4:
        prio 2: 7 routers
        prio 3: 8 routers
        prio 4: 3 routers
        prio 5: 6 routers
ovs-host5:
        prio 2: 7 routers
        prio 3: 4 routers
        prio 4: 7 routers
        prio 5: 6 routers


host 1 now back
ovs-host0:
        prio 1: 24 routers
ovs-host2:
        prio 2: 6 routers
        prio 3: 3 routers
        prio 4: 8 routers
        prio 5: 7 routers
ovs-host3:
        prio 2: 4 routers
        prio 3: 9 routers
        prio 4: 6 routers
        prio 5: 5 routers
ovs-host4:
        prio 2: 7 routers
        prio 3: 8 routers
        prio 4: 3 routers
        prio 5: 6 routers
ovs-host5:
        prio 2: 7 routers
        prio 3: 4 routers
        prio 4: 7 routers
        prio 5: 6 routers


host 2 now gone
ovs-host0:
        prio 2: 24 routers
ovs-host1:
        prio 1: 24 routers
ovs-host3:
        prio 3: 5 routers
        prio 4: 11 routers
        prio 5: 8 routers
ovs-host4:
        prio 3: 11 routers
        prio 4: 7 routers
        prio 5: 6 routers
ovs-host5:
        prio 3: 8 routers
        prio 4: 6 routers
        prio 5: 10 routers


host 2 now back
ovs-host0:
        prio 2: 24 routers
ovs-host1:
        prio 1: 24 routers
ovs-host3:
        prio 3: 5 routers
        prio 4: 11 routers
        prio 5: 8 routers
ovs-host4:
        prio 3: 11 routers
        prio 4: 7 routers
        prio 5: 6 routers
ovs-host5:
        prio 3: 8 routers
        prio 4: 6 routers
        prio 5: 10 routers


host 3 now gone
ovs-host0:
        prio 3: 24 routers
ovs-host1:
        prio 2: 24 routers
ovs-host2:
        prio 1: 24 routers
ovs-host4:
        prio 4: 13 routers
        prio 5: 11 routers
ovs-host5:
        prio 4: 11 routers
        prio 5: 13 routers


host 3 now back
ovs-host0:
        prio 3: 24 routers
ovs-host1:
        prio 2: 24 routers
ovs-host2:
        prio 1: 24 routers
ovs-host4:
        prio 4: 13 routers
        prio 5: 11 routers
ovs-host5:
        prio 4: 11 routers
        prio 5: 13 routers


host 4 now gone
ovs-host0:
        prio 4: 24 routers
ovs-host1:
        prio 3: 24 routers
ovs-host2:
        prio 2: 24 routers
ovs-host3:
        prio 1: 24 routers
ovs-host5:
        prio 5: 24 routers


host 4 now back
ovs-host0:
        prio 4: 24 routers
ovs-host1:
        prio 3: 24 routers
ovs-host2:
        prio 2: 24 routers
ovs-host3:
        prio 1: 24 routers
ovs-host5:
        prio 5: 24 routers


host 5 now gone
ovs-host0:
        prio 5: 24 routers
ovs-host1:
        prio 4: 24 routers
ovs-host2:
        prio 3: 24 routers
ovs-host3:
        prio 2: 24 routers
ovs-host4:
        prio 1: 24 routers


host 5 now back
ovs-host0:
        prio 5: 24 routers
ovs-host1:
        prio 4: 24 routers
ovs-host2:
        prio 3: 24 routers
ovs-host3:
        prio 2: 24 routers
ovs-host4:
        prio 1: 24 routers
```