Thank you for the suggestion.  I rolled back through our versions of nova.conf and it seems we've never had a cinder block in it.  I did populate it, restarted and the results are the same.  

/etc/nova/nova.conf
[DEFAULT]
cpu_allocation_ratio = 16.0
debug = true
enabled_apis = osapi_compute,metadata
metadata_proxy_shared_secret = <secret>
metadata_workers = 4
my_ip = 10.176.0.42
ram_allocation_ratio = 1.0
scheduler_host_subset_size = 2
transport_url = rabbit://openstack:<passwd>@mq-a01.xxx.cloud,openstack:<passwd>@mq-a02.xxx.cloud,openstack:<passwd>@mq-a03.xxx.cloud

[api]
auth_strategy = keystone

[api_database]
connection = mysql+pymysql://atx_nova_api:<passwd>@db-a00.xxx.cloud/atx_nova_api

[filter_scheduler]
enabled_filters = AvailabilityZoneFilter,ComputeCapabilitiesFilter,ComputeFilter,ImagePropertiesFilter,RamFilter,RetryFilter,ServerGroupAffinityFilter,ServerGroupAntiAffinityFilter
host_subset_size = 4
io_ops_weight_multiplier = 10.0
max_instances_per_host = 100
ram_weight_multiplier = 6.0
soft_affinity_weight_multiplier = 10.0
soft_anti_affinity_weight_multiplier = 10.0
weight_classes = nova.scheduler.weights.all_weighers

[database]
connection = mysql+pymysql://atx_nova:<passwd>@db-a00.xxx.cloud/atx_nova

[cinder]
catalog_info=volumev3:cinderv3:publicURL
os_region_name=atx
http_retries=3
auth_type = password
auth_url = https://cinder-s00.xxx.cloud
password = <password>
project_domain_name = Default
project_name = service
user_domain_name = Default
username = atx_cinder

[glance]
api_servers = https://glance-a00.xxx.cloud

[keystone_authtoken]
auth_type = password
www_authenticate_uri = https://keystone-a00.xxx.cloud
auth_url = https://keystone-a00.xxx.cloud
memcached_servers = memcached-a01.xxx.cloud:11211,memcached-a02.xxx.cloud:11211,memcached-a03.xxx.cloud:11211
password = <passwd>
project_domain_name = default
project_name = service
user_domain_name = default
username = atx_nova
service_token_roles_required = true

[neutron]
auth_type = password
auth_url = https://keystone-a00.xxx.cloud
metadata_proxy_shared_secret = <secret>
password = <passwd>
project_domain_name = Default
project_name = service
region_name = atx
service_metadata_proxy = true
#uses keystoneauth1: url = https://neutron-a00.xxx.cloud
user_domain_name = Default
username = atx_neutron

[oslo_concurrency]
lock_path = /var/lib/nova/tmp

[placement]
auth_type = password
auth_url = https://keystone-a00.xxx.cloud
os_region_name = atx
password = <passwd>
project_domain_name = Default
project_name = service
user_domain_name = Default
username = atx_placement

[scheduler]
discover_hosts_in_cells_interval = 300

[oslo_notifications_group]
driver = messaging
topics = notifications

[cache]
backend=oslo_cache.memcache_pool
enable = true

[oslo_messaging_rabbit]
amqp_durable_queues = true
rabbit_ha_queues = true
rabbit_retry_backoff = 2
rabbit_retry_interval = 1


/etc/cinder/cinder.conf (from the cinder node, of course)
[DEFAULT]
api_paste_config = /etc/cinder/api-paste.ini
auth_strategy = keystone
backup_driver = cinder.backup.drivers.ceph.CephBackupDriver
backup_ceph_conf=/etc/ceph/ceph.conf
backup_ceph_user = cinder
backup_ceph_chunk_size = 134217728
backup_ceph_pool = atx.cinder-backup
backup_ceph_stripe_unit = 0
backup_ceph_stripe_count = 0
restore_discard_excess_bytes = true
control_exchange = openstack
debug = true
enable_v2_api=false
enable_v3_api=true
default_availability_zone = nova
enabled_backends = cinder-volumes, sizeone
glance_api_servers = https://glance-a00.xxx.cloud
glance_api_version = 2
osapi_volume_listen = 0.0.0.0
osapi_volume_workers = 2
public_endpoint = https://cinder-a00.xxx.cloud
storage_availability_zone = nova
transport_url = rabbit://openstack:<password>@mq-a01.xxx.cloud,openstack:<password>@mq-a02.xxx.cloud,openstack:<password>@mq-a03.xxx.cloud

[nova]
region_name = nova

[database]
connection = mysql+pymysql://atx_cinder:<password>@db-a00.xxx.cloud/atx_cinder

[keystone_authtoken]
auth_type = password
auth_url = https://keystone-a00.xxx.cloud/v3
memcached_servers = memcached-a01.xxx.cloud:11211,memcached-a02.xxx.cloud:11211,memcached-a03.xxx.cloud:11211
project_domain_name = default
project_name = service
user_domain_name = default
username = atx_cinder
password = <password>
www_authenticate_uri = https://keystone-a00.xxx.cloud/v3
service_token_roles_required = true

[oslo_concurrency]
lock_path = /var/lib/cinder/tmp

[oslo_policy]
policy_file = /etc/cinder/policy.json

[cinder-volumes]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
volume_backend_name = cinder-volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_pool = atx.cinder-volumes
rbd_secret_uuid = <rbd-uuid>
rbd_user = cinder
report_discard_supported = true

[sizeone]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
volume_backend_name = sizeone
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_pool = atx.sizeone
rbd_secret_uuid = <rbd-uuid>
rbd_user = cinder
report_discard_supported = true

[oslo_messaging_rabbit]
amqp_durable_queues = true
rabbit_ha_queues = true
rabbit_retry_backoff = 2
rabbit_retry_interval = 1



Debug Logs
=============
Here are the cinder logs for the various commands:

I did: openstack add volume c331814d-b758-460e-9972-bc1e987b933d 0fc5bec7-9364-458c-a809-f38389890a60

==> /var/log/cinder/api.log <==
2020-05-14 09:42:27.856 2108738 DEBUG eventlet.wsgi.server [-] (2108738) accepted ('10.176.0.102', 56962) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
2020-05-14 09:42:27.879 2108738 DEBUG oslo_policy.policy [req-bfd335d4-4110-4f66-99f7-639a3474c253 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] The policy file /etc/cinder/policy.json could not be found. load_rules /usr/lib/python2.7/site-packages/oslo_policy/policy.py:559
2020-05-14 09:42:28.320 2108738 INFO cinder.api.openstack.wsgi [req-bfd335d4-4110-4f66-99f7-639a3474c253 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] GET http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60
2020-05-14 09:42:28.321 2108738 DEBUG cinder.api.openstack.wsgi [req-bfd335d4-4110-4f66-99f7-639a3474c253 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Empty body provided in request get_body /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:718
2020-05-14 09:42:28.322 2108738 DEBUG cinder.api.openstack.wsgi [req-bfd335d4-4110-4f66-99f7-639a3474c253 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Calling method 'show' _process_stack /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:872
2020-05-14 09:42:28.359 2108738 DEBUG oslo_db.sqlalchemy.engines [req-bfd335d4-4110-4f66-99f7-639a3474c253 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION _check_effective_sql_mode /usr/lib/python2.7/site-packages/oslo_db/sqlalchemy/engines.py:308
2020-05-14 09:42:28.401 2108738 INFO cinder.volume.api [req-bfd335d4-4110-4f66-99f7-639a3474c253 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Volume info retrieved successfully.
2020-05-14 09:42:28.429 2108738 INFO cinder.api.openstack.wsgi [req-bfd335d4-4110-4f66-99f7-639a3474c253 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 returned with HTTP 200
2020-05-14 09:42:28.430 2108738 INFO eventlet.wsgi.server [req-bfd335d4-4110-4f66-99f7-639a3474c253 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] 10.176.2.12,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 HTTP/1.1" status: 200  len: 1359 time: 0.5711679
2020-05-14 09:42:28.526 2108741 DEBUG eventlet.wsgi.server [-] (2108741) accepted ('10.176.0.102', 57036) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
2020-05-14 09:42:28.527 2108741 INFO cinder.api.openstack.wsgi [-] GET http://cinder-a00.xxx.cloud/
2020-05-14 09:42:28.528 2108741 DEBUG cinder.api.openstack.wsgi [-] Empty body provided in request get_body /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:718
2020-05-14 09:42:28.528 2108741 DEBUG cinder.api.openstack.wsgi [-] Calling method 'all' _process_stack /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:872
2020-05-14 09:42:28.529 2108741 INFO cinder.api.openstack.wsgi [-] http://cinder-a00.xxx.cloud/ returned with HTTP 300
2020-05-14 09:42:28.529 2108741 INFO eventlet.wsgi.server [-] 10.176.0.42,10.176.0.102 "GET / HTTP/1.1" status: 300  len: 576 time: 0.0025601
2020-05-14 09:42:28.552 2108741 INFO eventlet.wsgi.server [-] 10.176.0.42,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 HTTP/1.1" status: 401  len: 367 time: 0.0057251
2020-05-14 09:42:28.558 2108741 INFO eventlet.wsgi.server [-] 10.176.0.42,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 HTTP/1.1" status: 401  len: 367 time: 0.0033181


I did: openstack server add volume allalal peter-test-001

==> /var/log/cinder/api.log <==
2020-05-14 09:38:10.071 2108741 DEBUG eventlet.wsgi.server [-] (2108741) accepted ('10.176.0.102', 60460) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
2020-05-14 09:38:10.078 2108741 INFO cinder.api.openstack.wsgi [req-de2acbcb-18fd-4717-97be-27db471d58a4 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] GET http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/peter-test-001
2020-05-14 09:38:10.079 2108741 DEBUG cinder.api.openstack.wsgi [req-de2acbcb-18fd-4717-97be-27db471d58a4 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Empty body provided in request get_body /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:718
2020-05-14 09:38:10.080 2108741 DEBUG cinder.api.openstack.wsgi [req-de2acbcb-18fd-4717-97be-27db471d58a4 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Calling method 'show' _process_stack /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:872
2020-05-14 09:38:10.117 2108741 INFO cinder.api.openstack.wsgi [req-de2acbcb-18fd-4717-97be-27db471d58a4 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/peter-test-001 returned with HTTP 404
2020-05-14 09:38:10.119 2108741 INFO eventlet.wsgi.server [req-de2acbcb-18fd-4717-97be-27db471d58a4 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] 10.176.2.12,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/peter-test-001 HTTP/1.1" status: 404  len: 396 time: 0.0468712
2020-05-14 09:38:10.128 2108741 INFO cinder.api.openstack.wsgi [req-4d7f3d94-e886-428f-a118-7baa67ec0058 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] GET http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/detail?all_tenants=1&name=peter-test-001
2020-05-14 09:38:10.128 2108741 DEBUG cinder.api.openstack.wsgi [req-4d7f3d94-e886-428f-a118-7baa67ec0058 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Empty body provided in request get_body /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:718
2020-05-14 09:38:10.129 2108741 DEBUG cinder.api.openstack.wsgi [req-4d7f3d94-e886-428f-a118-7baa67ec0058 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Calling method 'detail' _process_stack /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:872
2020-05-14 09:38:10.130 2108741 DEBUG cinder.volume.api [req-4d7f3d94-e886-428f-a118-7baa67ec0058 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Searching by: MultiDict([('display_name', u'peter-test-001'), (u'all_tenants', 1)]). get_all /usr/lib/python2.7/site-packages/cinder/volume/api.py:615
2020-05-14 09:38:10.167 2108741 INFO cinder.volume.api [req-4d7f3d94-e886-428f-a118-7baa67ec0058 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Get all volumes completed successfully.
2020-05-14 09:38:10.174 2108741 INFO cinder.api.openstack.wsgi [req-4d7f3d94-e886-428f-a118-7baa67ec0058 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/detail?all_tenants=1&name=peter-test-001 returned with HTTP 200
2020-05-14 09:38:10.175 2108741 INFO eventlet.wsgi.server [req-4d7f3d94-e886-428f-a118-7baa67ec0058 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] 10.176.2.12,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/detail?all_tenants=1&name=peter-test-001 HTTP/1.1" status: 200  len: 1362 time: 0.0509570
2020-05-14 09:38:10.256 2108741 DEBUG eventlet.wsgi.server [-] (2108741) accepted ('10.176.0.102', 60480) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
2020-05-14 09:38:10.257 2108741 INFO cinder.api.openstack.wsgi [-] GET http://cinder-a00.xxx.cloud/
2020-05-14 09:38:10.258 2108741 DEBUG cinder.api.openstack.wsgi [-] Empty body provided in request get_body /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:718
2020-05-14 09:38:10.258 2108741 DEBUG cinder.api.openstack.wsgi [-] Calling method 'all' _process_stack /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:872
2020-05-14 09:38:10.259 2108741 INFO cinder.api.openstack.wsgi [-] http://cinder-a00.xxx.cloud/ returned with HTTP 300
2020-05-14 09:38:10.260 2108741 INFO eventlet.wsgi.server [-] 10.176.0.42,10.176.0.102 "GET / HTTP/1.1" status: 300  len: 576 time: 0.0026481
2020-05-14 09:38:10.282 2108741 INFO eventlet.wsgi.server [-] 10.176.0.42,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 HTTP/1.1" status: 401  len: 367 time: 0.0067141
2020-05-14 09:38:10.289 2108741 INFO eventlet.wsgi.server [-] 10.176.0.42,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 HTTP/1.1" status: 401  len: 367 time: 0.0034928



I did: openstack volume show 0fc5bec7-9364-458c-a809-f38389890a60

==> /var/log/cinder/api.log <==
2020-05-14 09:45:29.058 2108741 DEBUG eventlet.wsgi.server [-] (2108741) accepted ('10.176.0.102', 46052) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
2020-05-14 09:45:29.160 2108741 INFO cinder.api.openstack.wsgi [req-9d01db0a-6cf0-4734-9a75-7fa4178d934f 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] GET http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60
2020-05-14 09:45:29.161 2108741 DEBUG cinder.api.openstack.wsgi [req-9d01db0a-6cf0-4734-9a75-7fa4178d934f 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Empty body provided in request get_body /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:718
2020-05-14 09:45:29.161 2108741 DEBUG cinder.api.openstack.wsgi [req-9d01db0a-6cf0-4734-9a75-7fa4178d934f 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Calling method 'show' _process_stack /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:872
2020-05-14 09:45:29.207 2108741 INFO cinder.volume.api [req-9d01db0a-6cf0-4734-9a75-7fa4178d934f 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Volume info retrieved successfully.
2020-05-14 09:45:29.230 2108741 INFO cinder.api.openstack.wsgi [req-9d01db0a-6cf0-4734-9a75-7fa4178d934f 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 returned with HTTP 200
2020-05-14 09:45:29.231 2108741 INFO eventlet.wsgi.server [req-9d01db0a-6cf0-4734-9a75-7fa4178d934f 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] 10.176.2.12,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 HTTP/1.1" status: 200  len: 1359 time: 0.1718330


I did: openstack volume show peter-test-001

==> /var/log/cinder/api.log <==
2020-05-14 09:48:36.878 2108738 DEBUG eventlet.wsgi.server [-] (2108738) accepted ('10.176.0.102', 35738) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
2020-05-14 09:48:37.071 2108738 INFO cinder.api.openstack.wsgi [req-311e1f36-59d1-4c80-9755-3a4f82834e9a 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] GET http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/peter-test-001
2020-05-14 09:48:37.072 2108738 DEBUG cinder.api.openstack.wsgi [req-311e1f36-59d1-4c80-9755-3a4f82834e9a 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Empty body provided in request get_body /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:718
2020-05-14 09:48:37.072 2108738 DEBUG cinder.api.openstack.wsgi [req-311e1f36-59d1-4c80-9755-3a4f82834e9a 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Calling method 'show' _process_stack /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:872
2020-05-14 09:48:37.120 2108738 INFO cinder.api.openstack.wsgi [req-311e1f36-59d1-4c80-9755-3a4f82834e9a 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/peter-test-001 returned with HTTP 404
2020-05-14 09:48:37.122 2108738 INFO eventlet.wsgi.server [req-311e1f36-59d1-4c80-9755-3a4f82834e9a 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] 10.176.2.12,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/peter-test-001 HTTP/1.1" status: 404  len: 396 time: 0.2422359
2020-05-14 09:48:37.132 2108738 INFO cinder.api.openstack.wsgi [req-cd4f0648-5ddf-4d21-af20-23dfe4fd6d17 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] GET http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/detail?all_tenants=1&name=peter-test-001
2020-05-14 09:48:37.133 2108738 DEBUG cinder.api.openstack.wsgi [req-cd4f0648-5ddf-4d21-af20-23dfe4fd6d17 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Empty body provided in request get_body /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:718
2020-05-14 09:48:37.133 2108738 DEBUG cinder.api.openstack.wsgi [req-cd4f0648-5ddf-4d21-af20-23dfe4fd6d17 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Calling method 'detail' _process_stack /usr/lib/python2.7/site-packages/cinder/api/openstack/wsgi.py:872
2020-05-14 09:48:37.135 2108738 DEBUG cinder.volume.api [req-cd4f0648-5ddf-4d21-af20-23dfe4fd6d17 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Searching by: MultiDict([('display_name', u'peter-test-001'), (u'all_tenants', 1)]). get_all /usr/lib/python2.7/site-packages/cinder/volume/api.py:615
2020-05-14 09:48:37.177 2108738 INFO cinder.volume.api [req-cd4f0648-5ddf-4d21-af20-23dfe4fd6d17 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] Get all volumes completed successfully.
2020-05-14 09:48:37.188 2108738 INFO cinder.api.openstack.wsgi [req-cd4f0648-5ddf-4d21-af20-23dfe4fd6d17 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] http://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/detail?all_tenants=1&name=peter-test-001 returned with HTTP 200
2020-05-14 09:48:37.189 2108738 INFO eventlet.wsgi.server [req-cd4f0648-5ddf-4d21-af20-23dfe4fd6d17 497a19c5da034431b6c01d5d8f523968 1134ba6f01ef43f49961aa6bbe49f5bc - default default] 10.176.2.12,10.176.0.102 "GET /v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/detail?all_tenants=1&name=peter-test-001 HTTP/1.1" status: 200  len: 1362 time: 0.0612681





Observation
==============

The server add volume command seems to go fine when running openstack --debug until this:

GET call to volumev3 for https://cinder-a00.xxx.cloud/v3/1134ba6f01ef43f49961aa6bbe49f5bc/volumes/0fc5bec7-9364-458c-a809-f38389890a60 used request id req-99a6d42f-5cb0-4329-a685-b03951d8070d
REQ: curl -g -i -X POST https://nova-a00.xxx.cloud/v2.1/servers/c331814d-b758-460e-9972-bc1e987b933d/os-volume_attachments -H "User-Agent: python-novaclient" -H "Content-Type: application/json" -H "Accept: application/json" -H "X-Auth-Token: {SHA1}ca107041b2dd669a5841d1f98fe8ea545e8c78a0" -d '{"volumeAttachment": {"volumeId": "0fc5bec7-9364-458c-a809-f38389890a60"}}'
https://nova-a00.xxx.cloud:443 "POST /v2.1/servers/c331814d-b758-460e-9972-bc1e987b933d/os-volume_attachments HTTP/1.1" 500 210
RESP: [500] Openstack-Api-Version: compute 2.1 X-Openstack-Nova-Api-Version: 2.1 Vary: OpenStack-API-Version, X-OpenStack-Nova-API-Version Content-Type: application/json; charset=UTF-8 Content-Length: 210 X-Openstack-Request-Id: req-7b031061-45b9-4ad7-9fd0-c5d45a4bb838 X-Compute-Request-Id: req-7b031061-45b9-4ad7-9fd0-c5d45a4bb838 Date: Thu, 14 May 2020 14:53:11 GMT 
RESP BODY: {"computeFault": {"message": "Unexpected API Error. Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible.\n<class 'cinderclient.exceptions.Unauthorized'>", "code": 500}}

Doesn't the excerpt of the debug show it's talking fine to cinder but when it connects to nova to update the instance we trip over permissions?

Side note: I wonder if this worked with Queens where nova may have still been using volumev2/cinderv2 and the steps were different.