We are using ML2 midonet and have just upgraded our dev kit to Ocata. We are getting the following stack traces in the logs everytime we do something.
Everything seems to be working fine though from what I can tell.
Example of an error:
2018-02-28 09:36:25.666 23926 ERROR neutron.plugins.ml2.ovo_rpc [req-205d2290-93c3-4eb6-be57-c0584848e02c 02eaaab9ba6c48d6a5140d3f3acc1cb3 fa1f4b47be1242e48a75971cdc259bbe - - -] This handler is supposed to handle AFTER events, as in 'AFTER it's committed', not BEFORE. Offending resource event: port, after_delete. Location:
File "/usr/lib/python2.7/dist-packages/eventlet/greenpool.py", line 82, in _spawn_n_impl
func(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/eventlet/wsgi.py", line 719, in process_request
proto.__init__(sock, address, self)
File "/usr/lib/python2.7/SocketServer.py", line 652, in __init__
self.handle()
File "/usr/lib/python2.7/BaseHTTPServer.py", line 340, in handle
self.handle_one_request()
File "/usr/lib/python2.7/dist-packages/eventlet/wsgi.py", line 384, in handle_one_request
self.handle_one_response()
File "/usr/lib/python2.7/dist-packages/eventlet/wsgi.py", line 481, in handle_one_response
result = self.application(self.environ, start_response)
File "/usr/lib/python2.7/dist-packages/paste/urlmap.py", line 206, in __call__
return app(environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/oslo_middleware/base.py", line 126, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1299, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1263, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/oslo_middleware/base.py", line 126, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1299, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1263, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/oslo_middleware/request_id.py", line 37, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1299, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1263, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/oslo_middleware/catch_errors.py", line 41, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1299, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1263, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/keystonemiddleware/auth_token/__init__.py", line 335, in __call__
response = req.get_response(self._app)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1299, in send
application, catch_exc_info=False)
File "/usr/lib/python2.7/dist-packages/webob/request.py", line 1263, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__
return resp(environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__
return resp(environ, start_response)
File "/usr/lib/python2.7/dist-packages/routes/middleware.py", line 136, in __call__
response = self.app(environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__
return resp(environ, start_response)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/api/v2/resource.py", line 93, in resource
result = method(request=request, **args)
File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 562, in delete
return self._delete(request, id, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/db/api.py", line 91, in wrapped
return f(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/oslo_db/api.py", line 139, in wrapper
return f(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/db/api.py", line 126, in wrapped
return f(*dup_args, **dup_kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 590, in _delete
obj_deleter(request.context, id, **kwargs)
File "/usr/lib/python2.7/dist-packages/oslo_log/helpers.py", line 48, in wrapper
return method(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/db/api.py", line 166, in wrapped
return method(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/db/api.py", line 91, in wrapped
return f(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/oslo_db/api.py", line 139, in wrapper
return f(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/db/api.py", line 126, in wrapped
return f(*dup_args, **dup_kwargs)
File "/usr/lib/python2.7/dist-packages/midonet/neutron/services/l3/l3_midonet.py", line 170, in delete_router
super(MidonetL3ServicePlugin, self).delete_router(context, id)
File "/usr/lib/python2.7/dist-packages/neutron/db/l3_db.py", line 1817, in delete_router
super(L3_NAT_db_mixin, self).delete_router(context, id)
File "/usr/lib/python2.7/dist-packages/neutron/db/api.py", line 166, in wrapped
return method(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/db/l3_db.py", line 544, in delete_router
self._delete_current_gw_port(context, id, router, None)
File "/usr/lib/python2.7/dist-packages/neutron/db/l3_db.py", line 424, in _delete_current_gw_port
admin_ctx, gw_port_id, l3_port_check=False)
File "/usr/lib/python2.7/dist-packages/neutron/common/utils.py", line 787, in inner
return f(self, context, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/db/api.py", line 166, in wrapped
return method(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1554, in delete_port
context, port, router_ids, bound_mech_contexts)
File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1563, in _post_delete_port
registry.notify(resources.PORT, events.AFTER_DELETE, self, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/callbacks/registry.py", line 44, in notify
_get_callback_manager().notify(resource, event, trigger, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/db/api.py", line 177, in wrapped
return f(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/callbacks/manager.py", line 120, in notify
errors = self._notify_loop(resource, event, trigger, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/callbacks/manager.py", line 146, in _notify_loop
callback(resource, event, trigger, **kwargs)
File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/ovo_rpc.py", line 77, in handle_event
if self._is_session_semantic_violated(context, resource, event):
File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/ovo_rpc.py", line 61, in _is_session_semantic_violated
stack = traceback.extract_stack()
a plan is to fix it by converting our l3 plugin to an l3 flavor. cf. bug 1643830