prometheus openstack exporter missing object store metrics

[joek-office]

**Bug Report**

What happened:
When Openstack deployed with kolla-ansible is connected directly to ceph rgw service instead over openstack swift service, the metrics for object storage from prometheus openstack exporter was disabled.

What you expected to happen:
i expected, that in both cases:
- enable_ceph_rgw: "yes"
- enable_swift: "yes"
the container configuration for prometheus openstack exporter will not disable the metrics for object storage (--disable-service.object-store).

How to reproduce it (minimal and precise):
deploy openstack with kolla ansible with folowing options:
enable_swift: "no" (default)
enable_ceph_rgw: "yes"

and connected ceph with radosgw for object storage.

**Environment**:
* OS (e.g. from /etc/os-release):
  NAME="Rocky Linux"
  VERSION="9.3 (Blue Onyx)"

* Kernel (e.g. `uname -a`):
Linux ctrlhost1 5.14.0-362.24.1.el9_3.0.1.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Apr 4 22:31:43 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

* Docker version if applicable (e.g. `docker version`):
Client: Docker Engine - Community
 Version: 26.1.0
 API version: 1.45
 Go version: go1.21.9
 Git commit: 9714adc
 Built: Mon Apr 22 17:08:20 2024
 OS/Arch: linux/amd64
 Context: default

Server: Docker Engine - Community
 Engine:
  Version: 26.1.0
  API version: 1.45 (minimum version 1.24)
  Go version: go1.21.9
  Git commit: c8af8eb
  Built: Mon Apr 22 17:06:36 2024
  OS/Arch: linux/amd64
  Experimental: false
 containerd:
  Version: 1.6.31
  GitCommit: e377cd56a71523140ca6ae87e30244719194a521
 runc:
  Version: 1.1.12
  GitCommit: v1.1.12-0-g51d5e94
 docker-init:
  Version: 0.19.0
  GitCommit: de40ad0

* Are you using official images from Docker Hub or self built?
Images from quay.io

* Share your inventory file, globals.yml and other configuration files if relevant
-----------
---
workaround_ansible_issue_8743: yes
config_strategy: "COPY_ALWAYS"
kolla_base_distro: "rocky"
openstack_release: "2024.1"
node_custom_config: "/etc/kolla/config"
enable_docker_repo: false
docker_registry: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_PULL_QUAY_IO') }}"
docker_registry_insecure: "yes"
docker_registry_username: kolla-ansible
docker_namespace: "openstack.kolla"
docker_configure_for_zun: "no" # NOTE(msn): Disabled as currently broken in recent releases
network_interface: "bond0"
swift_storage_interface: "bond1"
tunnel_interface: "lo"
tunnel_interface_address: "{{ bgp_loopback_ip }}"
neutron_no_external_interface: "yes"
neutron_plugin_agent: "ovn"
neutron_ipam_driver: "internal"
neutron_enable_rolling_upgrade: "yes"
openvswitch_system_id: "{{ ansible_fqdn }}"
openvswitch_hostname: "{{ ansible_fqdn }}"
bgp_loopback_interface: "lo"
ovn_bgp_agent_driver: "nb_ovn_bgp_driver"
ovn_bgp_agent_exposing_method: "vrf"
keepalived_virtual_router_id: "51"
enable_container_healthchecks: "yes"
kolla_enable_tls_internal: "yes"
kolla_enable_tls_external: "yes"
kolla_external_fqdn_cert: "{{ kolla_certificates_dir }}/os-external-haproxy.pem"
kolla_internal_fqdn_cert: "{{ kolla_certificates_dir }}/os-internal-haproxy.pem"
kolla_copy_ca_into_containers: "yes"
openstack_cacert: "/etc/pki/tls/certs/ca-bundle.crt"
kolla_enable_tls_backend: "yes"
kolla_verify_tls_backend: "yes"
kolla_tls_backend_cert: "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"
kolla_tls_backend_key: "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"
openstack_region_name: "{{ site_name_full_upper }}"
enable_openstack_core: "yes"
enable_glance: "{{ enable_openstack_core | bool }}"
enable_hacluster: "yes"
enable_haproxy: "yes"
enable_keepalived: "{{ enable_haproxy | bool }}"
enable_keystone: "{{ enable_openstack_core | bool }}"
enable_mariadb: "yes"
enable_memcached: "yes"
enable_neutron: "{{ enable_openstack_core | bool }}"
enable_nova: "{{ enable_openstack_core | bool }}"
enable_central_logging: "no"
enable_ceph_rgw: "yes"
enable_ceph_rgw_loadbalancer: "no"
enable_cinder: "yes"
enable_cinder_backup: "yes"
enable_cloudkitty: "yes"
enable_fluentd: "yes"
enable_heat: "{{ enable_openstack_core | bool }}"
enable_horizon: "no"
enable_influxdb: "{{ enable_cloudkitty | bool and cloudkitty_storage_backend == 'influxdb' }}"
enable_kuryr: "no" # NOTE(msn): Disabled as currently broken in recent releases
enable_masakari: "yes"
enable_neutron_bgpvpn: "yes"
enable_neutron_port_forwarding: "yes"
enable_nova_ssh: "yes"
enable_octavia: "yes"
enable_opensearch: "{{ enable_central_logging | bool or enable_osprofiler | bool or (enable_cloudkitty | bool and cloudkitty_storage_backend == 'elasticsearch') }}"
enable_opensearch_dashboards: "{{ enable_opensearch | bool }}"
enable_opensearch_dashboards_external: "{{ enable_opensearch_dashboards | bool }}"
enable_frr: "yes"
enable_openvswitch: "{{ enable_neutron | bool and neutron_plugin_agent != 'linuxbridge' }}"
enable_ovn: "{{ enable_neutron | bool and neutron_plugin_agent == 'ovn' }}"
enable_ovn_bgp_agent: "yes"
enable_placement: "{{ enable_nova | bool or enable_zun | bool }}"
enable_prometheus: "yes"
enable_skyline: "yes"
enable_zun: "no" #NOTE(msn): Disabled as currently broken in recent releases
external_ceph_cephx_enabled: "yes"
cinder_ceph_backends:
  - name: "rbd_nvme"
    cluster: "ceph"
    pool_name: "rbd-nvme"
    user: "{{ ceph_cinder_user }}"
    keyring: "{{ ceph_cinder_keyring }}"
    secret_uuid: "{{ rbd_secret_uuid }}"
    enabled: "{{ cinder_backend_ceph | bool }}"
  - name: "rbd_ssd"
    cluster: "ceph"
    pool_name: "rbd-ssd"
    user: "{{ ceph_cinder_user }}"
    keyring: "{{ ceph_cinder_keyring }}"
    secret_uuid: "{{ rbd_secret_uuid }}"
    enabled: "{{ cinder_backend_ceph | bool }}"
  - name: "rbd_hdd"
    cluster: "ceph"
    pool_name: "rbd-hdd"
    user: "{{ ceph_cinder_user }}"
    keyring: "{{ ceph_cinder_keyring }}"
    secret_uuid: "{{ rbd_secret_uuid }}"
    enabled: "{{ cinder_backend_ceph | bool }}"
  - name: "rbd_hdd_ec_k2m1"
    cluster: "ceph"
    pool_name: "rbd-hdd-ec-k2m1"
    user: "{{ ceph_cinder_user }}"
    keyring: "{{ ceph_cinder_keyring }}"
    secret_uuid: "{{ rbd_secret_uuid }}"
    enabled: "{{ cinder_backend_ceph | bool }}"
  - name: "rbd_backups"
    cluster: "ceph"
    pool_name: "backups"
    user: "{{ ceph_cinder_user }}"
    keyring: "{{ ceph_cinder_keyring }}"
    secret_uuid: "{{ rbd_secret_uuid }}"
    enabled: "{{ cinder_backend_ceph | bool }}"
cinder_backup_backend_ceph_name: "rbd_backups"
ceph_glance_user: "glance"
ceph_glance_keyring: "client.glance.keyring"
ceph_glance_pool_name: "images"
ceph_cinder_user: "cinder"
ceph_cinder_keyring: "client.cinder.keyring"
ceph_cinder_pool_name: "volumes"
ceph_cinder_backup_user: "cinder-backup"
ceph_cinder_backup_keyring: "client.cinder-backup.keyring"
ceph_cinder_backup_pool_name: "backups"
ceph_nova_user: "cinder"
ceph_nova_keyring: "{{ ceph_cinder_keyring }}"
ceph_nova_pool_name: "vms"
ceph_rgw_port: 9443
ceph_rgw_swift_compatibility: False
ceph_rgw_swift_account_in_url: True
glance_backend_ceph: "yes"
glance_backend_file: "no"
cinder_backend_ceph: "yes"
cinder_backup_driver: "ceph"
cloudkitty_fetcher_backend: "prometheus"
cloudkitty_prometheus_insecure_connections: "true"
cloudkitty_prometheus_url: "{{ prometheus_api_base }}/api/v1/" # SET BY NETBOX CONFIG CONTEXT
cloudkitty_collector_backend: "prometheus"
cloudkitty_storage_backend: "influxdb"
cloudkitty_influxdb_use_ssl: "true"
cloudkitty_influxdb_insecure_connections: "true"
nova_backend_ceph: "yes"
nova_compute_virt_type: "kvm"
neutron_ovn_distributed_fip: "yes"
neutron_ovn_dhcp_agent: "yes"
skyline_prometheus_user: "kolla-ansible"
skyline_console_port: "443"
skyline_custom_logos: ["asset", "favicon.ico"]
enable_prometheus_server: "no"
enable_prometheus_alertmanager: "no"
prometheus_openstack_exporter_disabled_object: ''
octavia_provider_drivers: "ovn:OVN provider"
octavia_provider_agents: "ovn"
octavia_auto_configure: no
--------------------

[OpenStack Infra (hudson-openstack)]

Fix proposed to branch: master
Review: https://review.opendev.org/c/openstack/kolla-ansible/+/922920