I tried to update to 3.4.1/stable and the error persists, always 2 out of 3 regions have an error, at times even all 3 have an error, we had to rollback to 3.4.0


follow the error below

2024-03-26 03:17:31 maasserver.ipc: [critical] Failed to update regiond's processes and endpoints; prod-web-maas-region3:pid=944396 record's may be out of date
	Traceback (most recent call last):
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/asyncioreactor.py", line 271, in _onTimer
	    self.runUntilCurrent()
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/base.py", line 991, in runUntilCurrent
	    call.func(*call.args, **call.kw)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 700, in errback
	    self._startRunCallbacks(fail)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 763, in _startRunCallbacks
	    self._runCallbacks()
	--- <exception caught here> ---
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 857, in _runCallbacks
	    current.result = callback(  # type: ignore[misc]
	  File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/ipc.py", line 622, in ignore_cancel
	    failure.trap(CancelledError)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/failure.py", line 451, in trap
	    self.raiseException()
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/failure.py", line 475, in raiseException
	    raise self.value.with_traceback(self.tb)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 244, in inContext
	    result = inContext.theWork()  # type: ignore[attr-defined]
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 260, in <lambda>
	    inContext.theWork = lambda: context.call(  # type: ignore[attr-defined]
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/context.py", line 117, in callWithContext
	    return self.currentContext().callWithContext(ctx, func, *args, **kw)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/twisted/python/context.py", line 82, in callWithContext
	    return func(*args, **kw)
	  File "/snap/maas/34087/lib/python3.10/site-packages/provisioningserver/utils/twisted.py", line 856, in callInContext
	    return func(*args, **kwargs)
	  File "/snap/maas/34087/lib/python3.10/site-packages/provisioningserver/utils/twisted.py", line 203, in wrapper
	    result = func(*args, **kwargs)
	  File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/utils/orm.py", line 771, in call_within_transaction
	    return func_outside_txn(*args, **kwargs)
	  File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/utils/orm.py", line 587, in retrier
	    return func(*args, **kwargs)
	  File "/usr/lib/python3.10/contextlib.py", line 79, in inner
	    return func(*args, **kwds)
	  File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/ipc.py", line 590, in _update
	    self._updateConnections(process, conn["rpc"]["connections"].copy())
	  File "/snap/maas/34087/lib/python3.10/site-packages/provisioningserver/utils/twisted.py", line 203, in wrapper
	    result = func(*args, **kwargs)
	  File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/ipc.py", line 518, in _updateConnections
	    db_conn, _ = self._registerConnection(
	  File "/snap/maas/34087/lib/python3.10/site-packages/provisioningserver/utils/twisted.py", line 203, in wrapper
	    result = func(*args, **kwargs)
	  File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/ipc.py", line 391, in _registerConnection
	    connection, created = RegionRackRPCConnection.objects.get_or_create(
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/manager.py", line 85, in manager_method
	    return getattr(self.get_queryset(), name)(*args, **kwargs)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/query.py", line 588, in get_or_create
	    return self.create(**params), True
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/query.py", line 453, in create
	    obj.save(force_insert=True, using=self.db)
	  File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/models/cleansave.py", line 48, in save
	    return super().save(*args, **kwargs)
	  File "/snap/maas/34087/lib/python3.10/site-packages/maasserver/models/timestampedmodel.py", line 78, in save
	    return super().save(*args, **kwargs)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/base.py", line 739, in save
	    self.save_base(using=using, force_insert=force_insert,
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/base.py", line 776, in save_base
	    updated = self._save_table(
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/base.py", line 881, in _save_table
	    results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/base.py", line 919, in _do_insert
	    return manager._insert(
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/manager.py", line 85, in manager_method
	    return getattr(self.get_queryset(), name)(*args, **kwargs)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/query.py", line 1270, in _insert
	    return query.get_compiler(using=using).execute_sql(returning_fields)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/models/sql/compiler.py", line 1416, in execute_sql
	    cursor.execute(sql, params)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 66, in execute
	    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
	    return executor(sql, params, many, context)
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 79, in _execute
	    with self.db.wrap_database_errors:
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/utils.py", line 90, in __exit__
	    raise dj_exc_value.with_traceback(traceback) from exc_value
	  File "/snap/maas/34087/usr/lib/python3/dist-packages/django/db/backends/utils.py", line 84, in _execute
	    return self.cursor.execute(sql, params)
	django.db.utils.IntegrityError: duplicate key value violates unique constraint "maasserver_regionrackrpcconne_endpoint_id_52afd589b0be380e_uniq"
	DETAIL:  Key (endpoint_id, rack_controller_id)=(51289, 5934) already exists.