System is running Rocky using RHEL and RHEL's OpenStack repositories
python-networking-vmware-nsx-2018.0.0-0.20180803224551.322d37b.el7ost.noarch
python2-vmware-nsxlib-13.0.1-0.20181004050247.2d9f856.el7ost.noarch
`router_distributed` was not defined anywhere so I've tried defining `router_distributed = false` in `[DEFAULT]` but error remains.
I'm attaching the initial log where Neutron tries to create the Metadata Proxy.
When trying to create a router manually the same error will occur.
I'm a bit lost where to start looking, any pointers would be great.
I've already grep'ed through the entire distribution to see where `router_distributed` is being used but it's hardly used anywhere at all and defaults safely to `false`...
log:
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy [req-6bc7b9bb-214c-4031-a8a2-f16db2bf6876 - - - - -] Exception Callback vmware_nsx.plugins.nsx_v.plugin.NsxVPluginV2._process_az_req
uest--9223372036854116211 failed with "no such option router_distributed in group [DEFAULT]" while creating internal edge for metadata service: CallbackFailure: Callback vmware_nsx.plugins.nsx_v.plug
in.NsxVPluginV2._process_az_request--9223372036854116211 failed with "no such option router_distributed in group [DEFAULT]"
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy Traceback (most recent call last):
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/vmware_nsx/plugins/nsx_v/md_proxy.py", line 457, in _setup_new_proxy_edge
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy allow_metadata=False)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/vmware_nsx/plugins/nsx_v/plugin.py", line 3194, in create_router
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy lrouter = super(NsxVPluginV2, self).create_router(context, router)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron/db/l3_db.py", line 1935, in create_router
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy router)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron/db/api.py", line 123, in wrapped
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy return method(*args, **kwargs)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron_lib/db/api.py", line 140, in wrapped
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy setattr(e, '_RETRY_EXCEEDED', True)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy self.force_reraise()
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy six.reraise(self.type_, self.value, self.tb)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron_lib/db/api.py", line 136, in wrapped
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy return f(*args, **kwargs)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_db/api.py", line 154, in wrapper
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy ectxt.value = e.inner_exc
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy self.force_reraise()
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy six.reraise(self.type_, self.value, self.tb)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_db/api.py", line 142, in wrapper
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy return f(*args, **kwargs)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron_lib/db/api.py", line 183, in wrapped
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy LOG.debug("Retry wrapper got retriable exception: %s", e)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy self.force_reraise()
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy six.reraise(self.type_, self.value, self.tb)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron_lib/db/api.py", line 179, in wrapped
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy return f(*dup_args, **dup_kwargs)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron/db/l3_db.py", line 253, in create_router
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy transaction=False)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron/db/_utils.py", line 60, in safe_creation
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy obj = create_fn()
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron/db/l3_db.py", line 233, in _create_router_db
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy router_id=router['id'], router_db=router_db)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron_lib/callbacks/registry.py", line 56, in notify
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy _get_callback_manager().notify(resource, event, trigger, **kwargs)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron_lib/db/utils.py", line 107, in _wrapped
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy raise db_exc.RetryRequest(e)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy self.force_reraise()
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy six.reraise(self.type_, self.value, self.tb)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron_lib/db/utils.py", line 102, in _wrapped
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy return function(*args, **kwargs)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy File "/usr/lib/python2.7/site-packages/neutron_lib/callbacks/manager.py", line 177, in notify
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy raise exceptions.CallbackFailure(errors=errors)
2019-03-04 13:32:04.915 21 ERROR vmware_nsx.plugins.nsx_v.md_proxy CallbackFailure: Callback vmware_nsx.plugins.nsx_v.plugin.NsxVPluginV2._process_az_request--9223372036854116211 failed with "no such
option router_distributed in group [DEFAULT]"
Additional Info:
VCSA 6.7.0.20000
NSX 6.4.4.11197766
ESXi 6.7.0 11675023
neutron.conf: //user: pass@rabbit: 5672/ nsx.plugins. nsx_v.plugin. NsxVPluginV2 lbaas.services. loadbalancer. plugin. LoadBalancerPlu ginv2,neutron_ fwaas.services. firewall. fwaas_plugin. FirewallPlugin //user: pass@mysql/ neutron authtoken] e_uri = http:// keystone keystone keystone _notifications] VMWareEdge: neutron_ lbaas.drivers. vmware. edge_driver_ v2.EdgeLoadBala ncerDriverV2: default fwaasDriver: vmware_ nsx.services. fwaas.nsx_ v.edge_ fwaas_driver. EdgeFwaasDriver :default nsx.services. fwaas.nsx_ v.edge_ fwaas_driver. EdgeFwaasDriver lbaas.drivers. vmware. edge_driver_ v2.EdgeLoadBala ncerDriverV2
[DEFAULT]
fatal_deprecations = false
debug = true
use_stderr = true
use_json = false
notification_format = cadf
transport_url = rabbit:
host = neutron
core_plugin = vmware_
service_plugins = neutron_
api_workers = 1
[database]
connection = mysql+pymysql:
[keystone_
www_authenticat
auth_url = http://
memcached_servers = memcached:11211
auth_type = password
project_domain_name = Default
user_domain_name = Default
project_name = service
username = user
password = pass
[nova]
auth_type = password
auth_url = http://
project_domain_name = default
user_domain_name = default
region_name = Dev
project_name = service
username = user
password = pass
[oslo_messaging
driver = messagingv2
[service_providers]
service_provider = LOADBALANCERV2:
service_provider = FIREWALL:
[fwaas]
driver = vmware_
enabled = true
[lbaas]
driver = neutron_
enabled = true
plugin.ini: missing_ data = true /1.2.3. 6 tunnels_ per_vnic = 110 compact: 0:1,vdr: compact: 0:1 proxy_netmask = 255.255.255.0 shared_ secret = pass distributed, exclusive initializer = true router_ appliance_ size = compact router_ appliance_ size = large 8.8.4.4, 1.1.1.1 insertion_ redirect_ all = true _to_all_ interfaces = true between_ tenants = true block_all = false
[DEFAULT]
nsx_l2gw_driver = NSX-V
[nsx_lsn]
sync_on_
[nsxv]
user = "admin"
password = "pass"
manager_uri = https:/
insecure = true
cluster_moid = domain-c7
datacenter_moid = datacenter-2
resource_pool_id = resgroup-16
datastore_id = datastore-11
external_network = dvportgroup-19
vdn_scope_id = vdnscope-1
dvs_id = dvs-17
maximum_
backup_edge_pool = service:
mgt_net_moid = dvportgroup-20
mgt_net_proxy_ips = 1.2.3.5
mgt_net_
nova_metadata_ips = 1.2.3.4
metadata_
metadata_insecure = true
tenant_router_types = shared,
dhcp_lease_time = 86400
metadata_
edge_ha = false
exclusive_
shared_
nameservers = 8.8.8.8,
service_
bind_floatingip
exclusive_dhcp_edge = false
share_edges_
use_default_