At Ussuri Neutron API has begun doing writes directly to the Southbound DB, there does not appear to be a accompanying RBAC role for this, so do we need to give it access to the private port currently reserved for ovn-northd? Example of an failed transaction: 2020-02-25 11:04:33.420 1520231 ERROR ovsdbapp.backend.ovs_idl.transaction [req-8315d356-f92f-4447-a47b-f724374cfc36 - - - - -] OVSDB Error: {"details":"RBAC rules for client \"juju-ef641e-1-lxd-2.maas\" role \"ovn-controller\" prohibit modification of table \"Chassis\".","error":"permission error"} 2020-02-25 11:04:33.420 1520231 ERROR ovsdbapp.backend.ovs_idl.transaction [req-fbf878ca-f0bc-465c-b173-882d695cb4aa 3ff519473176440bb9678c95051ed627 dd8f9f301d1e436d8d3a9b695537c897 - cb4c93ee9c98459c8cde54c2c8b0a829 cb4c93ee9c98459c8cde54c2c8b0a829] Traceback (most recent call last): File "/usr/lib/python3/dist-packages/ovsdbapp/backend/ovs_idl/connection.py", line 122, in run txn.results.put(txn.do_commit()) File "/usr/lib/python3/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 115, in do_commit raise RuntimeError(msg) RuntimeError: OVSDB Error: {"details":"RBAC rules for client \"juju-ef641e-1-lxd-2.maas\" role \"ovn-controller\" prohibit modification of table \"Chassis\".","error":"permission error"} 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command [req-fbf878ca-f0bc-465c-b173-882d695cb4aa 3ff519473176440bb9678c95051ed627 dd8f9f301d1e436d8d3a9b695537c897 - cb4c93ee9c98459c8cde54c2c8b0a829 cb4c93ee9c98459c8cde54c2c8b0a829] Error executing command: RuntimeError: OVSDB Error: {"details":"RBAC rules for client \"juju-ef641e-1-lxd-2.maas\" role \"ovn-controller\" prohibit modification of table \"Chassis\".","error":"permission error"} 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command Traceback (most recent call last): 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command File "/usr/lib/python3/dist-packages/ovsdbapp/backend/ovs_idl/command.py", line 40, in execute 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command t.add(self) 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command next(self.gen) 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command File "/usr/lib/python3/dist-packages/ovsdbapp/api.py", line 119, in transaction 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command del self._nested_txns_map[cur_thread_id] 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command File "/usr/lib/python3/dist-packages/ovsdbapp/api.py", line 69, in __exit__ 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command self.result = self.commit() 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command File "/usr/lib/python3/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 62, in commit 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command raise result.ex 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command File "/usr/lib/python3/dist-packages/ovsdbapp/backend/ovs_idl/connection.py", line 122, in run 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command txn.results.put(txn.do_commit()) 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command File "/usr/lib/python3/dist-packages/ovsdbapp/backend/ovs_idl/transaction.py", line 115, in do_commit 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command raise RuntimeError(msg) 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command RuntimeError: OVSDB Error: {"details":"RBAC rules for client \"juju-ef641e-1-lxd-2.maas\" role \"ovn-controller\" prohibit modification of table \"Chassis\".","error":"permission error"} 2020-02-25 11:04:33.421 1520231 ERROR ovsdbapp.backend.ovs_idl.command 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation [req-fbf878ca-f0bc-465c-b173-882d695cb4aa 3ff519473176440bb9678c95051ed627 dd8f9f301d1e436d8d3a9b695537c897 - cb4c93ee9c98459c8cde54c2c8b0a829 cb4c93ee9c98459c8cde54c2c8b0a829] POST failed.: neutron_lib.callbacks.exceptions.CallbackFailure: Callback neutron.services.segments.db._add_segment_host_mapping_for_segment--9223363248144570382 failed with "OVSDB Error: {"details":"RBAC rules for client \"juju-ef641e-1-lxd-2.maas\" role \"ovn-controller\" prohibit modification of table \"Chassis\".","error":"permission error"}" 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation Traceback (most recent call last): 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/pecan/core.py", line 683, in __call__ 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation self.invoke_controller(controller, args, kwargs, state) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/pecan/core.py", line 574, in invoke_controller 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation result = controller(*args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 139, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation setattr(e, '_RETRY_EXCEEDED', True) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise() 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise value 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 135, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_db/api.py", line 154, in wrapper 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation ectxt.value = e.inner_exc 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise() 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise value 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_db/api.py", line 142, in wrapper 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 183, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation LOG.debug("Retry wrapper got retriable exception: %s", e) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise() 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise value 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 179, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return f(*dup_args, **dup_kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/pecan_wsgi/controllers/utils.py", line 76, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/pecan_wsgi/controllers/resource.py", line 163, in post 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return self.create(resources) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/pecan_wsgi/controllers/resource.py", line 181, in create 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return {key: creator(*creator_args, **creator_kwargs)} 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/common/utils.py", line 685, in inner 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return f(self, context, *args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 233, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return method(*args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 139, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation setattr(e, '_RETRY_EXCEEDED', True) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise() 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise value 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 135, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_db/api.py", line 154, in wrapper 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation ectxt.value = e.inner_exc 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise() 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise value 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_db/api.py", line 142, in wrapper 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 183, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation LOG.debug("Retry wrapper got retriable exception: %s", e) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise() 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise value 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/api.py", line 179, in wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return f(*dup_args, **dup_kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/plugins/ml2/plugin.py", line 1047, in create_network 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation result, mech_context = self._create_network_db(context, network) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/plugins/ml2/plugin.py", line 1006, in _create_network_db 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation tenant_id) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/plugins/ml2/managers.py", line 218, in create_network_segments 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation segment_index) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/plugins/ml2/managers.py", line 201, in _add_network_segment 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation context, network_id, segment, segment_index) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron/db/segments_db.py", line 57, in add_network_segment 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation segment=netseg_obj) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/callbacks/registry.py", line 56, in notify 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation _get_callback_manager().notify(resource, event, trigger, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/utils.py", line 108, in _wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise db_exc.RetryRequest(e) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 220, in __exit__ 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise() 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/six.py", line 693, in reraise 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise value 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/db/utils.py", line 103, in _wrapped 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation return function(*args, **kwargs) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python3/dist-packages/neutron_lib/callbacks/manager.py", line 177, in notify 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation raise exceptions.CallbackFailure(errors=errors) 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation neutron_lib.callbacks.exceptions.CallbackFailure: Callback neutron.services.segments.db._add_segment_host_mapping_for_segment--9223363248144570382 failed with "OVSDB Error: {"details":"RBAC rules for client \"juju-ef641e-1-lxd-2.maas\" role \"ovn-controller\" prohibit modification of table \"Chassis\".","error":"permission error"}" 2020-02-25 11:04:33.486 1520231 ERROR neutron.pecan_wsgi.hooks.translation