Running Victoria release using openstack-ansible deployment and noticed when i create SRIOV instance using terraform and try to delete it failed with following error in neutron
[ml2]
type_drivers = flat,vlan,vxlan,local
tenant_network_types = vxlan,vlan
mechanism_drivers = linuxbridge,sriovnicswitch
# ML2 flat networks
extension_drivers = port_security,subnet_dns_publish_fixed_ip
root@ostack-phx-api-1-2-neutron-server-container-ff9f8108:~# journalctl -u neutron-server -n1000 | grep -i error
Jul 21 14:13:50 ostack-phx-api-1-2-neutron-server-container-ff9f8108 neutron-server[94]: 2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation [req-cc389662-0746-479a-af53-609f2e2efa3d 9e8c9cff22dcb519e7ea76e7dc9906702a07c48b334bc225ca218787fe99bbfe c7df41f0ec294154a33d3be5a523924c - 716be371313b47fe8d9c015cd747465a default] DELETE failed.: designateclient.exceptions.Forbidden: Forbidden
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation Traceback (most recent call last):
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/plugins/ml2/plugin.py", line 1848, in _pre_delete_port
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation registry.publish(
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/callbacks/registry.py", line 60, in publish
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation _get_callback_manager().publish(resource, event, trigger, payload=payload)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/callbacks/manager.py", line 149, in publish
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return self.notify(resource, event, trigger, payload=payload)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/utils.py", line 108, in _wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise db_exc.RetryRequest(e)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/six.py", line 703, in reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise value
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/utils.py", line 103, in _wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return function(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/callbacks/manager.py", line 174, in notify
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise exceptions.CallbackFailure(errors=errors)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation neutron_lib.callbacks.exceptions.CallbackFailure: Callback neutron.plugins.ml2.extensions.dns_integration._delete_port_in_external_dns_service-8757283612694 failed with "Forbidden"
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation During handling of the above exception, another exception occurred:
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation Traceback (most recent call last):
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/pecan/core.py", line 683, in __call__
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self.invoke_controller(controller, args, kwargs, state)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/pecan/core.py", line 574, in invoke_controller
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation result = controller(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 139, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation setattr(e, '_RETRY_EXCEEDED', True)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/six.py", line 703, in reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise value
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 135, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_db/api.py", line 154, in wrapper
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation ectxt.value = e.inner_exc
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/six.py", line 703, in reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise value
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_db/api.py", line 142, in wrapper
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 183, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation LOG.debug("Retry wrapper got retriable exception: %s", e)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/six.py", line 703, in reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise value
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 179, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return f(*dup_args, **dup_kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/pecan_wsgi/controllers/utils.py", line 76, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/pecan_wsgi/controllers/utils.py", line 115, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation f(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/pecan_wsgi/controllers/resource.py", line 85, in delete
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return self.plugin_deleter(*deleter_args)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/common/utils.py", line 681, in inner
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return f(self, context, *args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 233, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return method(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 139, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation setattr(e, '_RETRY_EXCEEDED', True)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/six.py", line 703, in reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise value
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 135, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_db/api.py", line 154, in wrapper
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation ectxt.value = e.inner_exc
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/six.py", line 703, in reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise value
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_db/api.py", line 142, in wrapper
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 183, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation LOG.debug("Retry wrapper got retriable exception: %s", e)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/six.py", line 703, in reraise
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise value
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/db/api.py", line 179, in wrapped
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return f(*dup_args, **dup_kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/plugins/ml2/plugin.py", line 1862, in delete_port
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation self._pre_delete_port(context, id, l3_port_check)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/plugins/ml2/plugin.py", line 1856, in _pre_delete_port
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise e.errors[0].error
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron_lib/callbacks/manager.py", line 197, in _notify_loop
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation callback(resource, event, trigger, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/plugins/ml2/extensions/dns_integration.py", line 553, in _delete_port_in_external_dns_service
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation _remove_data_from_external_dns_service(
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/plugins/ml2/extensions/dns_integration.py", line 491, in _remove_data_from_external_dns_service
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation dns_driver.delete_record_set(context, dns_domain, dns_name, records)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/services/externaldns/drivers/designate/driver.py", line 162, in delete_record_set
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation ids_to_delete = self._get_ids_ips_to_delete(
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/neutron/services/externaldns/drivers/designate/driver.py", line 178, in _get_ids_ips_to_delete
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation recordsets = designate_client.recordsets.list(
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/designateclient/v2/recordsets.py", line 69, in list
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation zone = v2_utils.resolve_by_name(self.client.zones.list, zone)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/designateclient/v2/utils.py", line 34, in resolve_by_name
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation results = func(criterion={"name": "%s" % name}, *args)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/designateclient/v2/zones.py", line 51, in list
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return self._get(url, response_key="zones")
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/designateclient/v2/base.py", line 30, in _get
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation resp, body = self.client.session.get(url, **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/keystoneauth1/adapter.py", line 395, in get
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation return self.request(url, 'GET', **kwargs)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation File "/openstack/venvs/neutron-22.1.2/lib/python3.8/site-packages/designateclient/v2/client.py", line 93, in request
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation raise exceptions.Forbidden(**response_payload)
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation designateclient.exceptions.Forbidden: Forbidden
2021-07-21 14:13:50.371 94 ERROR neutron.pecan_wsgi.hooks.translation
Do You have configured designate integration in the neutron-server's config? Like is described in https:/