SM:mainline:2743: Kilo : HA  neutron plugin not  working  neutron.manager ImportError: No module named neutron_plugin_contrail.plugins.opencontrail.contrail_plugin


1) Provision HA setup with Servermanager as follows
server-manager-client  display server --select id,cluster_id,roles,ip_address
+---------+----------------+---------------+----------------------------------------------------------------------------+
|    id   |   cluster_id   |   ip_address  |                                   roles                                    |
+---------+----------------+---------------+----------------------------------------------------------------------------+
| nodeg21 | cluster5sanity | 10.204.217.61 | [u'control', u'webui', u'openstack', u'database', u'collector', u'config'] |
| nodec38 | cluster5sanity | 10.204.217.23 |                                [u'compute']                                |
| nodeg17 | cluster5sanity | 10.204.217.57 | [u'control', u'webui', u'openstack', u'database', u'collector', u'config'] |
| nodec58 | cluster5sanity | 10.204.217.98 |                                [u'compute']                                |
| nodeg33 | cluster5sanity | 10.204.217.73 | [u'control', u'webui', u'openstack', u'database', u'collector', u'config'] |
+---------+----------------+---------------+----------------------------------------------------------------------------+
root@nodej8:~# 

keystone  endpoint-list | grep 5000
| b9fa502a65a541c682cf3bb651161274 | RegionOne |       http://10.204.217.176:5000/v2.0       |       http://10.204.217.176:5000/v2.0       |       http://10.204.217.176:35357/v2.0      | b8de74e170f640e382b5e4b40f1903c1 |


2) Neutron commands are not working on the nodes

root@nodeg21:/var/log/keystone# neutron net-list
An unknown exception occurred.



Neutron-Server log

16-09-29 03:48:42.629 30089 DEBUG keystoneclient.session [-] REQ: curl -g -i -X GET http://10.204.217.176:35357/v3/auth/tokens -H "X-Subject-Token: {SHA1}873b88b76e672a336436b1dfd8efe16d15fb0bce" -H "User-Agent: python-keystoneclient" -H "Accept: application/json" -H "X-Auth-Token: {SHA1}f963dc95e057362e5f0071f5897c1e9afab4c9a6" _http_log_request /usr/lib/python2.7/dist-packages/keystoneclient/session.py:195
2016-09-29 03:48:42.714 30089 DEBUG keystoneclient.session [-] RESP: [200] content-length: 7234 x-subject-token: {SHA1}873b88b76e672a336436b1dfd8efe16d15fb0bce vary: X-Auth-Token x-distribution: Ubuntu date: Thu, 29 Sep 2016 10:48:42 GMT content-type: application/json x-openstack-request-id: req-c00ba4df-e110-4eb2-b9b9-036376155362
RESP BODY: {"token": {"methods": ["password", "token"], "roles": [{"id": "ddac72831c6143898e7fcd265ff2c059", "name": "admin"}, {"id": "9d0a9350418a44ebb243bae02c244534", "name": "ResellerAdmin"}], "expires_at": "2016-09-29T11:48:41.000000Z", "project": {"domain": {"id": "default", "name": "Default"}, "id": "4c702850d4ec496b9de939c0badbbf63", "name": "services"}, "catalog": "<removed>", "extras": {}, "user": {"domain": {"id": "default", "name": "Default"}, "id": "e41494991b154f919d945a6040ada4fb", "name": "ceilometer"}, "audit_ids": ["zj33VR7gTMWuUJzD3LURpg"], "issued_at": "2016-09-29T10:48:41.061280"}}
 _http_log_response /usr/lib/python2.7/dist-packages/keystoneclient/session.py:223
2016-09-29 03:48:42.718 30089 ERROR neutron.api.v2.resource [req-975f308b-3c39-4f83-b7c5-fcd8859f3045 ] index failed
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource Traceback (most recent call last):
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/resource.py", line 83, in resource
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource     result = method(request=request, **args)
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 319, in index
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource     return self._items(request, True, parent_id)
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 249, in _items
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource     obj_list = obj_getter(request.context, **kwargs)
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin_base.py", line 62
2, in get_floatingips
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource     return self._list_resource('floatingip', context, filters, fields)
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin.py", line 314, in _list_resource
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource     fields=fields, obj_name=res_type)
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin.py", line 237, in _transform_response
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource     plugin_base._raise_contrail_error(info, obj_name)
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin_base.py", line 111, in _raise_contrail_error
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource     raise exc.NeutronException(**info)
2016-09-29 03:48:42.718 30089 TRACE neutron.api.v2.resource NeutronException: An unknown exception occurred.






2016-09-29 03:46:36.667 28515 DEBUG neutron.service [-] ******************************************************************************** log_opt_values /usr/lib/python2.7/dist-packages/oslo_config/cfg.py:2197
2016-09-29 03:46:36.667 28515 INFO neutron.common.config [-] Config paste file: /etc/neutron/api-paste.ini
2016-09-29 03:46:36.678 28515 DEBUG oslo_concurrency.lockutils [-] Lock "manager" acquired by "_create_instance" :: waited 0.000s inner /usr/lib/python2.7/dist-packages/oslo_concurrency/lockutils.py:444
2016-09-29 03:46:36.679 28515 INFO neutron.manager [-] Loading core plugin: neutron_plugin_contrail.plugins.opencontrail.contrail_plugin.NeutronPluginContrailCoreV2
2016-09-29 03:46:36.679 28515 ERROR neutron.manager [-] Error loading plugin by name, No 'neutron.core_plugins' driver found, looking for 'neutron_plugin_contrail.plugins.opencontrail.contrail_plugin.NeutronPluginContrailCoreV2'
2016-09-29 03:46:36.679 28515 TRACE neutron.manager Traceback (most recent call last):
2016-09-29 03:46:36.679 28515 TRACE neutron.manager   File "/usr/lib/python2.7/dist-packages/neutron/manager.py", line 138, in _get_plugin_instance
2016-09-29 03:46:36.679 28515 TRACE neutron.manager     plugin_class = importutils.import_class(plugin_provider)
2016-09-29 03:46:36.679 28515 TRACE neutron.manager   File "/usr/lib/python2.7/dist-packages/oslo_utils/importutils.py", line 27, in import_class
2016-09-29 03:46:36.679 28515 TRACE neutron.manager     __import__(mod_str)
2016-09-29 03:46:36.679 28515 TRACE neutron.manager ImportError: No module named neutron_plugin_contrail.plugins.opencontrail.contrail_plugin
2016-09-29 03:46:36.679 28515 TRACE neutron.manager
2016-09-29 03:46:36.680 28515 ERROR neutron.manager [-] Error loading plugin by class, No module named neutron_plugin_contrail.plugins.opencontrail.contrail_plugin
2016-09-29 03:46:36.680 28515 TRACE neutron.manager Traceback (most recent call last):
2016-09-29 03:46:36.680 28515 TRACE neutron.manager   File "/usr/lib/python2.7/dist-packages/neutron/manager.py", line 138, in _get_plugin_instance
2016-09-29 03:46:36.680 28515 TRACE neutron.manager     plugin_class = importutils.import_class(plugin_provider)
2016-09-29 03:46:36.680 28515 TRACE neutron.manager   File "/usr/lib/python2.7/dist-packages/oslo_utils/importutils.py", line 27, in import_class
2016-09-29 03:46:36.680 28515 TRACE neutron.manager     __import__(mod_str)
2016-09-29 03:46:36.680 28515 TRACE neutron.manager ImportError: No module named neutron_plugin_contrail.plugins.opencontrail.contrail_plugin
2016-09-29 03:46:36.680 28515 TRACE neutron.manager
2016-09-29 03:46:36.680 28515 DEBUG oslo_concurrency.lockutils [-] Lock "manager" released by "_create_instance" :: held 0.002s inner /usr/lib/python2.7/dist-packages/oslo_concurrency/lockutils.py:456
2016-09-29 03:46:36.680 28515 ERROR neutron.common.config [-] Unable to load neutron from configuration file /etc/neutron/api-paste.ini.
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config Traceback (most recent call last):
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/common/config.py", line 227, in load_paste_app
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     app = deploy.loadapp("config:%s" % config_path, name=app_name)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 247, in loadapp
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return loadobj(APP, uri, name=name, **kw)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 272, in loadobj
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return context.create()
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 710, in create
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return self.object_type.invoke(self)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 144, in invoke
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     **context.local_conf)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/util.py", line 55, in fix_call
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     val = callable(*args, **kw)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/urlmap.py", line 28, in urlmap_factory

2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     app = loader.get_app(app_name, global_conf=global_conf)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 350, in get_app
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     name=name, global_conf=global_conf).create()
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 710, in create
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return self.object_type.invoke(self)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 144, in invoke
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     **context.local_conf)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/util.py", line 55, in fix_call
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     val = callable(*args, **kw)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/auth.py", line 71, in pipeline_factory
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     app = loader.get_app(pipeline[-1])
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 350, in get_app
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     name=name, global_conf=global_conf).create()
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 710, in create
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return self.object_type.invoke(self)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 146, in invoke
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return fix_call(context.object, context.global_conf, **context.local_conf)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/paste/deploy/util.py", line 55, in fix_call
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     val = callable(*args, **kw)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/router.py", line 71, in factory
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return cls(**local_config)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/router.py", line 75, in __init__
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     plugin = manager.NeutronManager.get_plugin()
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/manager.py", line 222, in get_plugin
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return weakref.proxy(cls.get_instance().plugin)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/manager.py", line 216, in get_instance
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     cls._create_instance()
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/oslo_concurrency/lockutils.py", line 445, in inner
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     return f(*args, **kwargs)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/manager.py", line 202, in _create_instance
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     cls._instance = cls()
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/manager.py", line 117, in __init__
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     plugin_provider)
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config   File "/usr/lib/python2.7/dist-packages/neutron/manager.py", line 142, in _get_plugin_instance
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config     raise ImportError(_("Plugin not found."))
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config ImportError: Plugin not found.
2016-09-29 03:46:36.680 28515 TRACE neutron.common.config
2016-09-29 03:46:36.682 28515 ERROR neutron.service [-] Unrecoverable error: please check log for details.
2016-09-29 03:46:36.682 28515 TRACE neutron.service Traceback (most recent call last):
2016-09-29 03:46:36.682 28515 TRACE neutron.service   File "/usr/lib/python2.7/dist-packages/neutron/service.py", line 103, in serve_wsgi
2016-09-29 03:46:36.682 28515 TRACE neutron.service     service.start()
2016-09-29 03:46:36.682 28515 TRACE neutron.service   File "/usr/lib/python2.7/dist-packages/neutron/service.py", line 74, in start
2016-09-29 03:46:36.682 28515 TRACE neutron.service     self.wsgi_app = _run_wsgi(self.app_name)
2016-09-29 03:46:36.682 28515 TRACE neutron.service   File "/usr/lib/python2.7/dist-packages/neutron/service.py", line 169, in _run_wsgi 2016-09-29 03:46:36.682 28515 TRACE neutron.service     app = config.load_paste_app(app_name)
2016-09-29 03:46:36.682 28515 TRACE neutron.service   File "/usr/lib/python2.7/dist-packages/neutron/common/config.py", line 234, in load_paste_app
2016-09-29 03:46:36.682 28515 TRACE neutron.service     raise RuntimeError(msg)
2016-09-29 03:46:36.682 28515 TRACE neutron.service RuntimeError: Unable to load neutron from configuration file /etc/neutron/api-paste.ini.
2016-09-29 03:46:36.682 28515 TRACE neutron.service
2016-09-29 03:46:37.078 28558 CRITICAL neutron.manager [-] Neutron core_plugin not configured!
2016-09-29 03:46:37.079 28558 ERROR neutron.service [-] Unrecoverable error: please check log for details.