Hello
OpenStack Ussuri (Ubuntu 20.04)
Periodically, when deleting instances, we encounter an error
```
ERROR oslo_messaging.rpc.server [req-cf1f6938-7e78-4f0e-84c1-485103077d71 1ff8a0897796c01a27e3a529f8595fc4d063703ded21c6e82a758967287149ea 6a4a603df34642e38d50a7a93c0068a1 - 5f5>
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 2203, in _instance_update
instance_ref = model_query(context, models.Instance,
File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/orm.py", line 52, in update_on_match
return update_match.update_on_match(
File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/update_match.py", line 194, in update_on_match
raise NoRowsMatched("Zero rows matched for %d attempts" % attempts)
oslo_db.sqlalchemy.update_match.NoRowsMatched: Zero rows matched for 3 attempts
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/nova/conductor/manager.py", line 139, in _object_dispatch
return getattr(target, method)(*args, **kwargs)
File "/usr/lib/python3/dist-packages/oslo_versionedobjects/base.py", line 226, in wrapper
return fn(self, *args, **kwargs)
File "/usr/lib/python3/dist-packages/nova/objects/instance.py", line 836, in save
old_ref, inst_ref = db.instance_update_and_get_original(
File "/usr/lib/python3/dist-packages/nova/db/api.py", line 675, in instance_update_and_get_original
rv = IMPL.instance_update_and_get_original(context, instance_uuid, values,
File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 180, in wrapper
return f(*args, **kwargs)
File "/usr/lib/python3/dist-packages/oslo_db/api.py", line 154, in wrapper
ectxt.value = e.inner_exc
File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
self.force_reraise()
File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
six.reraise(self.type_, self.value, self.tb)
File "/usr/lib/python3/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/lib/python3/dist-packages/oslo_db/api.py", line 142, in wrapper
return f(*args, **kwargs)
File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 223, in wrapped
return f(context, *args, **kwargs)
File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 2122, in instance_update_and_get_original
return (copy.copy(instance_ref), _instance_update(
File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 2264, in _instance_update
raise exc(**exc_props)
nova.exception.UnexpectedDeletingTaskStateError: Conflict updating instance adac16cb-ae4b-4c12-a7d9-112e40b41eac. Expected: {'task_state': ['powering-off']}. Actual: {'task_state': 'deleting'}
ERROR oslo_messaging.rpc.server Traceback (most recent call last):
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming
ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_messaging/rpc/dispatcher.py", line 276, in dispatch
ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_messaging/rpc/dispatcher.py", line 196, in _do_dispatch
ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/exception_wrapper.py", line 77, in wrapped
ERROR oslo_messaging.rpc.server _emit_exception_notification(
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
ERROR oslo_messaging.rpc.server self.force_reraise()
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/six.py", line 703, in reraise
ERROR oslo_messaging.rpc.server raise value
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/exception_wrapper.py", line 69, in wrapped
ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 165, in decorated_function
ERROR oslo_messaging.rpc.server LOG.info("Task possibly preempted: %s",
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
ERROR oslo_messaging.rpc.server self.force_reraise()
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/six.py", line 703, in reraise
ERROR oslo_messaging.rpc.server raise value
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 159, in decorated_function
ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/compute/utils.py", line 1456, in decorated_function
ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 216, in decorated_function
ERROR oslo_messaging.rpc.server compute_utils.add_instance_fault_from_exc(context,
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
ERROR oslo_messaging.rpc.server self.force_reraise()
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/six.py", line 703, in reraise
ERROR oslo_messaging.rpc.server raise value
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 205, in decorated_function
ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 3170, in stop_instance
ERROR oslo_messaging.rpc.server do_stop_instance()
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_concurrency/lockutils.py", line 359, in inner
ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/compute/manager.py", line 3162, in do_stop_instance
ERROR oslo_messaging.rpc.server instance.save(expected_task_state=expected_task_state)
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_versionedobjects/base.py", line 209, in wrapper
ERROR oslo_messaging.rpc.server updates, result = self.indirection_api.object_action(
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/conductor/rpcapi.py", line 247, in object_action
ERROR oslo_messaging.rpc.server return cctxt.call(context, 'object_action', objinst=objinst,
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_messaging/rpc/client.py", line 177, in call
ERROR oslo_messaging.rpc.server self.transport._send(self.target, msg_ctxt, msg,
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_messaging/transport.py", line 124, in _send
ERROR oslo_messaging.rpc.server return self._driver.send(target, ctxt, message,
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 680, in send
ERROR oslo_messaging.rpc.server return self._send(target, ctxt, message, wait_for_reply, timeout,
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 672, in _send
ERROR oslo_messaging.rpc.server raise result
ERROR oslo_messaging.rpc.server nova.exception_Remote.UnexpectedDeletingTaskStateError_Remote: Conflict updating instance adac16cb-ae4b-4c12-a7d9-112e40b41eac. Expected: {'task_state': ['powering-off']}. Actual: {'task_state': 'deleting'}
ERROR oslo_messaging.rpc.server Traceback (most recent call last):
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 2203, in _instance_update
ERROR oslo_messaging.rpc.server instance_ref = model_query(context, models.Instance,
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/orm.py", line 52, in update_on_match
ERROR oslo_messaging.rpc.server return update_match.update_on_match(
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/update_match.py", line 194, in update_on_match
ERROR oslo_messaging.rpc.server raise NoRowsMatched("Zero rows matched for %d attempts" % attempts)
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server oslo_db.sqlalchemy.update_match.NoRowsMatched: Zero rows matched for 3 attempts
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred:
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server Traceback (most recent call last):
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/conductor/manager.py", line 139, in _object_dispatch
ERROR oslo_messaging.rpc.server return getattr(target, method)(*args, **kwargs)
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_versionedobjects/base.py", line 226, in wrapper
ERROR oslo_messaging.rpc.server return fn(self, *args, **kwargs)
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/objects/instance.py", line 836, in save
ERROR oslo_messaging.rpc.server old_ref, inst_ref = db.instance_update_and_get_original(
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/db/api.py", line 675, in instance_update_and_get_original
ERROR oslo_messaging.rpc.server rv = IMPL.instance_update_and_get_original(context, instance_uuid, values,
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 180, in wrapper
ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_db/api.py", line 154, in wrapper
ERROR oslo_messaging.rpc.server ectxt.value = e.inner_exc
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
ERROR oslo_messaging.rpc.server self.force_reraise()
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
ERROR oslo_messaging.rpc.server six.reraise(self.type_, self.value, self.tb)
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/six.py", line 703, in reraise
ERROR oslo_messaging.rpc.server raise value
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/oslo_db/api.py", line 142, in wrapper
ERROR oslo_messaging.rpc.server return f(*args, **kwargs)
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 223, in wrapped
ERROR oslo_messaging.rpc.server return f(context, *args, **kwargs)
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 2122, in instance_update_and_get_original
ERROR oslo_messaging.rpc.server return (copy.copy(instance_ref), _instance_update(
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 2264, in _instance_update
ERROR oslo_messaging.rpc.server raise exc(**exc_props)
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server nova.exception.UnexpectedDeletingTaskStateError: Conflict updating instance adac16cb-ae4b-4c12-a7d9-112e40b41eac. Expected: {'task_state': ['powering-off']}. Actual: {'task_state': 'deleting'}
ERROR oslo_messaging.rpc.server
ERROR oslo_messaging.rpc.server
```
The instance is eventually deleted, but notifications about this are not created and the dns record remains in the designate forever
```
dpkg -l | grep nova
ii nova-common 2:21.2.4-0ubuntu2.5 all OpenStack Compute - common files
ii nova-compute-kvm 2:21.2.4-0ubuntu2.5 all OpenStack Compute - compute node (KVM)
ii nova-compute-libvirt 2:21.2.4-0ubuntu2.5 all OpenStack Compute - compute node libvirt support
ii python3-nova 2:21.2.4-0ubuntu2.5 all OpenStack Compute Python 3 libraries
ii python3-novaclient 2:17.0.0-0ubuntu1 all client library for OpenStack Compute API - 3.x
```
