SERIES=mantic # jammy focal

# LXD VM

lxc launch --vm --config limits.cpu=2 --config limits.memory=4GiB ubuntu:$SERIES cinder-$SERIES
lxc shell cinder-$SERIES

# Apt

apt remove -y unattended-upgrades && apt update

# Network

echo '127.0.0.1 controller' >>/etc/hosts

# MySQL

apt install -y mysql-server python3-pymysql

# RabbitMQ

apt install -y rabbitmq-server

rabbitmqctl add_user openstack RABBIT_PASS
rabbitmqctl set_permissions openstack ".*" ".*" ".*"

# Keystone

cat <<EOF | mysql
CREATE DATABASE keystone;
CREATE USER 'keystone'@'localhost' IDENTIFIED BY 'KEYSTONE_DBPASS';
GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost';
EOF

apt install -y keystone

@ /etc/keystone/keystone.conf
# [database]
# connection = mysql+pymysql://keystone:KEYSTONE_DBPASS@controller/keystone
#
# [token]
# provider = fernet


sed '/^\[database\]/,/^$/ s#connection = .*#connection = mysql+pymysql://keystone:KEYSTONE_DBPASS@controller/keystone#' -i /etc/keystone/keystone.conf

sed '/^\[token\]/ a provider = fernet' -i /etc/keystone/keystone.conf

...

   # Mantic has some exceptions ignored in these commands, but worked fine (exit code 0):
	# keystone-manage db-sync
		Exception ignored in: <function _removeHandlerRef at 0x79e0817f9b20>
		Traceback (most recent call last):
		  File "/usr/lib/python3.11/logging/__init__.py", line 855, in _removeHandlerRef
		  File "/usr/lib/python3.11/logging/__init__.py", line 235, in _acquireLock
		  File "/usr/lib/python3.11/threading.py", line 164, in acquire
		  File "/usr/lib/python3/dist-packages/eventlet/green/thread.py", line 34, in get_ident
		AttributeError: 'NoneType' object has no attribute 'getcurrent'

	# fernet_setup
		Exception ignored in: <function _removeHandlerRef at 0x79b4752fdb20>
		Traceback (most recent call last):
		  File "/usr/lib/python3.11/logging/__init__.py", line 855, in _removeHandlerRef
		  File "/usr/lib/python3.11/logging/__init__.py", line 235, in _acquireLock
		  File "/usr/lib/python3.11/threading.py", line 164, in acquire
		  File "/usr/lib/python3/dist-packages/eventlet/green/thread.py", line 34, in get_ident
		AttributeError: 'NoneType' object has no attribute 'getcurrent'

	# credential_setup
		Exception ignored in: <function _removeHandlerRef at 0x772c8ef8db20>
		Traceback (most recent call last):
		  File "/usr/lib/python3.11/logging/__init__.py", line 855, in _removeHandlerRef
		  File "/usr/lib/python3.11/logging/__init__.py", line 235, in _acquireLock
		  File "/usr/lib/python3.11/threading.py", line 164, in acquire
		  File "/usr/lib/python3/dist-packages/eventlet/green/thread.py", line 34, in get_ident
		AttributeError: 'NoneType' object has no attribute 'getcurrent'


   
su -s /bin/sh -c "keystone-manage db_sync" keystone # exit code 0 even with exception on mantic

keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone # exit code 0
keystone-manage credential_setup --keystone-user keystone --keystone-group keystone # exit code 0
keystone-manage bootstrap --bootstrap-password ADMIN_PASS \
  --bootstrap-admin-url http://controller:5000/v3/ \
  --bootstrap-internal-url http://controller:5000/v3/ \
  --bootstrap-public-url http://controller:5000/v3/ \
  --bootstrap-region-id RegionOne
 
echo 'ServerName controller' >>/etc/apache2/apache2.conf
systemctl restart apache2.service

# Openstack Client

apt install -y python3-openstackclient

cat <<EOF >openstack.rc
export OS_USERNAME=admin
export OS_PASSWORD=ADMIN_PASS
export OS_PROJECT_NAME=admin
export OS_USER_DOMAIN_NAME=Default
export OS_PROJECT_DOMAIN_NAME=Default
export OS_AUTH_URL=http://controller:5000/v3
export OS_IDENTITY_API_VERSION=3
EOF

source openstack.rc

openstack project create --domain default \
  --description "Service Project" service
  
+-------------+----------------------------------+
| Field       | Value                            |
+-------------+----------------------------------+
| description | Service Project                  |
| domain_id   | default                          |
| enabled     | True                             |
| id          | 229ef5671d0b4136b32f1a60584ab725 |
| is_domain   | False                            |
| name        | service                          |
| options     | {}                               |
| parent_id   | default                          |
| tags        | []                               |
+-------------+----------------------------------+

# openstack project list
+----------------------------------+---------+
| ID                               | Name    |
+----------------------------------+---------+
| 0e204837047a4323b8f57ed23bffa4f8 | admin   |
| 229ef5671d0b4136b32f1a60584ab725 | service |
+----------------------------------+---------+

# Cinder

cat <<EOF | mysql
CREATE DATABASE cinder;
CREATE USER 'cinder'@'localhost' IDENTIFIED BY 'CINDER_DBPASS';
GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'localhost';
EOF

openstack user create --domain default --password CINDER_USERPASS cinder
openstack role add --project service --user cinder admin

openstack service create --name cinderv2 \
  --description "OpenStack Block Storage" volumev2
openstack service create --name cinderv3 \
  --description "OpenStack Block Storage" volumev3

openstack endpoint create --region RegionOne \
  volumev2 public http://controller:8776/v2/%\(project_id\)s
openstack endpoint create --region RegionOne \
  volumev2 internal http://controller:8776/v2/%\(project_id\)s
openstack endpoint create --region RegionOne \
  volumev2 admin http://controller:8776/v2/%\(project_id\)s

openstack endpoint create --region RegionOne \
  volumev3 public http://controller:8776/v3/%\(project_id\)s
openstack endpoint create --region RegionOne \
  volumev3 internal http://controller:8776/v3/%\(project_id\)s
openstack endpoint create --region RegionOne \
  volumev3 admin http://controller:8776/v3/%\(project_id\)s

apt install -y cinder-api cinder-scheduler

@ /etc/cinder/cinder.conf

sed '/^\[DEFAULT\]/ a my_ip = 127.0.0.1' -i /etc/cinder/cinder.conf
sed '/^\[DEFAULT\]/ a transport_url = rabbit://openstack:RABBIT_PASS@controller' -i /etc/cinder/cinder.conf
sed '/^\[DEFAULT\]/ a auth_strategy = keystone' -i /etc/cinder/cinder.conf
sed '/^\[database\]/,/^$/ s#connection = .*#connection = mysql+pymysql://cinder:CINDER_DBPASS@controller/cinder#' -i /etc/cinder/cinder.conf

cat <<EOF >>/etc/cinder/cinder.conf
[keystone_authtoken]
www_authenticate_uri = http://controller:5000
auth_url = http://controller:5000
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = cinder
password = CINDER_USERPASS

[oslo_concurrency]
lock_path = /var/lib/cinder/tmp
EOF

su -s /bin/sh -c "cinder-manage db sync" cinder

systemctl restart cinder-scheduler.service apache2.service

# Cinder Volume

truncate -s 5GiB /test.img

mknod --mode 0660 /dev/loop-cinder b 7 42
losetup /dev/loop-cinder /test.img

pvcreate /dev/loop-cinder
vgcreate cinder-volumes /dev/loop-cinder

@ /etc/cinder/cinder.conf

# Mantic/Jammy: important (switch back from lioadm to tgtadm); safe on Focal.

sed 's/^iscsi_helper = .*/iscsi_helper = tgtadm/' -i /etc/cinder/cinder.conf

sed '/^\[DEFAULT\]/ a enabled_backends = lvm' -i /etc/cinder/cinder.conf

cat <<EOF >>/etc/cinder/cinder.conf
[lvm]
lvm_type = default
volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver
volume_group = cinder-volumes
target_protocol = iscsi
target_helper = tgtadm
EOF

...

apt install -y cinder-volume tgt thin-provisioning-tools

echo 'include /var/lib/cinder/volumes/*' >/etc/tgt/conf.d/cinder.conf

systemctl restart tgt.service cinder-volume.service


# Cinder Client

apt install -y python3-cinderclient

# cinder list
+----+--------+------+------+----------------+-------------+----------+-------------+
| ID | Status | Name | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+----+--------+------+------+----------------+-------------+----------+-------------+
+----+--------+------+------+----------------+-------------+----------+-------------+

cinder create --name test-volume 1

# cinder list
+--------------------------------------+-----------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status    | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+-----------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | available | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+-----------+-------------+------+----------------+-------------+----------+-------------+

  # Mantic requires attachment to set as in-use; safe on Jammy; skip on Focal (needs a Server)
  
cinder attachment-create test-volume

# cinder attachment-list
+--------------------------------------+--------------------------------------+----------+-----------+
| ID                                   | Volume ID                            | Status   | Server ID |
+--------------------------------------+--------------------------------------+----------+-----------+
| 90704b06-cf65-473c-adef-7e8c8b5c8b2c | 17c4f736-058b-4cfc-9864-a64ab0995957 | reserved | -         |
+--------------------------------------+--------------------------------------+----------+-----------+

cinder reset-state --state in-use test-volume

# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+


FIRST: REPRODUCER
-----

# Reproducer (45-second startup delay in tgt.service)

FILE=/etc/systemd/system/tgt.service.d/start-delay.conf
mkdir -p $(dirname $FILE)
cat <<EOF > $FILE
[Service]
ExecStartPre=$(which sleep) 45
EOF
systemctl daemon-reload

# systemctl show tgt.service | grep ^ExecStartPre=
ExecStartPre={ path=/usr/bin/sleep ; argv[]=/usr/bin/sleep 45 ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }

# date; systemctl restart cinder-volume.service tgt.service; date

... Notice that cinder-volume.service is Started _BEFORE_ tgt.service
... And the 'tgtadm' error _IS_ seen in the logs.

mantic:

	root@cinder-mantic:~# date; systemctl restart cinder-volume.service tgt.service; date
	Tue Apr 30 15:54:43 UTC 2024
	Tue Apr 30 15:55:29 UTC 2024

	root@cinder-mantic:~# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -3
	Apr 30 15:54:43 cinder-mantic systemd[1]: Started cinder-volume.service - OpenStack Cinder Volume.
	Apr 30 15:54:44 cinder-mantic systemd[1]: Starting tgt.service - (i)SCSI target daemon...
	Apr 30 15:55:29 cinder-mantic systemd[1]: Started tgt.service - (i)SCSI target daemon.

	root@cinder-mantic:~# grep 'ERROR cinder.volume.manager' /var/log/cinder/cinder-volume.log | grep tgtadm
	2024-04-30 15:54:54.603 13692 ERROR cinder.volume.manager     (out, err) = cinder.privsep.targets.tgt.tgtadm_show()
	2024-04-30 15:54:54.603 13692 ERROR cinder.volume.manager Command: tgtadm --lld iscsi --op show --mode target
	2024-04-30 15:54:54.603 13692 ERROR cinder.volume.manager Stderr: 'tgtadm: failed to send request hdr to tgt daemon, Transport endpoint is not connected\n'

	root@cinder-mantic:~# cinder list
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
	| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
	| 45b39ae0-2552-4703-a664-ae8de77a23a1 | error  | test-volume | 1    | True           | __DEFAULT__ | false    |             |
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+


jammy:

	root@cinder-jammy:~# date; systemctl restart cinder-volume.service tgt.service; date
	Tue Apr 30 15:54:45 UTC 2024
	Tue Apr 30 15:55:30 UTC 2024

	root@cinder-jammy:~# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -3
	Apr 30 15:54:45 cinder-jammy systemd[1]: Started OpenStack Cinder Volume.
	Apr 30 15:54:45 cinder-jammy systemd[1]: Starting (i)SCSI target daemon...
	Apr 30 15:55:30 cinder-jammy systemd[1]: Started (i)SCSI target daemon.

	root@cinder-jammy:~# grep 'ERROR cinder.volume.manager' /var/log/cinder/cinder-volume.log | grep tgtadm
	2024-04-30 15:54:55.918 14576 ERROR cinder.volume.manager     (out, err) = cinder.privsep.targets.tgt.tgtadm_show()
	2024-04-30 15:54:55.918 14576 ERROR cinder.volume.manager Command: tgtadm --lld iscsi --op show --mode target
	2024-04-30 15:54:55.918 14576 ERROR cinder.volume.manager Stderr: 'tgtadm: failed to send request hdr to tgt daemon, Transport endpoint is not connected\n'

	root@cinder-jammy:~# cinder list
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
	| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
	| 88024eca-203c-4f8f-9ff0-4de704041a2f | error  | test-volume | 1    | True           | __DEFAULT__ | false    |             |
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+

focal:

	root@cinder-focal:~# date; systemctl restart cinder-volume.service tgt.service; date
	Tue Apr 30 15:54:46 UTC 2024
	Tue Apr 30 15:55:31 UTC 2024

	root@cinder-focal:~# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -3
	Apr 30 15:54:46 cinder-focal systemd[1]: Started OpenStack Cinder Volume.
	Apr 30 15:54:46 cinder-focal systemd[1]: Starting (i)SCSI target daemon...
	Apr 30 15:55:31 cinder-focal systemd[1]: Started (i)SCSI target daemon.

	root@cinder-focal:~# grep 'ERROR cinder.volume.manager' /var/log/cinder/cinder-volume.log | grep tgtadm
	2024-04-30 15:54:55.830 15834 ERROR cinder.volume.manager     (out, err) = cinder.privsep.targets.tgt.tgtadm_show()
	2024-04-30 15:54:55.830 15834 ERROR cinder.volume.manager Command: tgtadm --lld iscsi --op show --mode target
	2024-04-30 15:54:55.830 15834 ERROR cinder.volume.manager Stderr: 'tgtadm: failed to send request hdr to tgt daemon, Transport endpoint is not connected\n'

	root@cinder-focal:~# cinder list
	+--------------------------------------+--------+-------------+------+-------------+----------+-------------+
	| ID                                   | Status | Name        | Size | Volume Type | Bootable | Attached to |
	+--------------------------------------+--------+-------------+------+-------------+----------+-------------+
	| 5fa41f17-afaf-4a56-8494-8c9246531523 | error  | test-volume | 1    | __DEFAULT__ | false    |             |
	+--------------------------------------+--------+-------------+------+-------------+----------+-------------+

Status is error:

# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | error  | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+

# Undo error state

cinder reset-state --state in-use test-volume

# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 17c4f736-058b-4cfc-9864-a64ab0995957 | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+

NEXT: VERIFY SRUS
----


Before:

mantic:

	root@cinder-mantic:~# dpkg -l | grep cinder
	ii  cinder-api                      2:23.0.0-0ubuntu1.1                     all          Cinder storage service - API server
	ii  cinder-common                   2:23.0.0-0ubuntu1.1                     all          Cinder storage service - common files
	ii  cinder-scheduler                2:23.0.0-0ubuntu1.1                     all          Cinder storage service - Scheduler server
	ii  cinder-volume                   2:23.0.0-0ubuntu1.1                     all          Cinder storage service - Volume server
	ii  python3-cinder                  2:23.0.0-0ubuntu1.1                     all          Cinder Python 3 libraries
	ii  python3-cinderclient            1:9.4.0-0ubuntu1                        all          Python bindings to the OpenStack Volume API - Python 3.x

	root@cinder-mantic:~# systemctl show cinder-volume.service | grep Wants=
	Wants=network-online.target

jammy:

	root@cinder-jammy:~# dpkg -l | grep cinder
	ii  cinder-api                      2:20.3.1-0ubuntu1.1                     all          Cinder storage service - API server
	ii  cinder-common                   2:20.3.1-0ubuntu1.1                     all          Cinder storage service - common files
	ii  cinder-scheduler                2:20.3.1-0ubuntu1.1                     all          Cinder storage service - Scheduler server
	ii  cinder-volume                   2:20.3.1-0ubuntu1.1                     all          Cinder storage service - Volume server
	ii  python3-cinder                  2:20.3.1-0ubuntu1.1                     all          Cinder Python 3 libraries
	ii  python3-cinderclient            1:8.3.0-0ubuntu1                        all          Python bindings to the OpenStack Volume API - Python 3.x

	root@cinder-jammy:~# systemctl show cinder-volume.service | grep Wants=
	Wants=network-online.target

focal:

	root@cinder-focal:~# dpkg -l | grep cinder
	ii  cinder-api                     2:16.4.2-0ubuntu2.4               all          Cinder storage service - API server
	ii  cinder-common                  2:16.4.2-0ubuntu2.4               all          Cinder storage service - common files
	ii  cinder-scheduler               2:16.4.2-0ubuntu2.4               all          Cinder storage service - Scheduler server
	ii  cinder-volume                  2:16.4.2-0ubuntu2.4               all          Cinder storage service - Volume server
	ii  python3-cinder                 2:16.4.2-0ubuntu2.4               all          Cinder Python 3 libraries
	ii  python3-cinderclient           1:7.0.0-0ubuntu1                  all          Python bindings to the OpenStack Volume API - Python 3.x

	root@cinder-focal:~# systemctl show cinder-volume.service | grep Wants=
	Wants=network-online.target



Enable -proposed:

echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-proposed main universe" >/etc/apt/sources.list.d/proposed.list
apt update

cat <<EOF >/etc/apt/preferences.d/proposed
Package: *
Pin: release a=$(lsb_release -cs)-proposed
Pin-Priority: 400
EOF

Upgrade cinder packages:

apt install -t $(lsb_release -cs)-proposed -y cinder-volume

After:

mantic:

	root@cinder-mantic:~# apt-cache policy cinder-volume
	cinder-volume:
	  Installed: 2:23.0.0-0ubuntu1.2
	  Candidate: 2:23.0.0-0ubuntu1.2
	  Version table:
	 *** 2:23.0.0-0ubuntu1.2 500
		400 http://archive.ubuntu.com/ubuntu mantic-proposed/main amd64 Packages
		100 /var/lib/dpkg/status
	     2:23.0.0-0ubuntu1.1 500
		500 http://archive.ubuntu.com/ubuntu mantic-updates/main amd64 Packages
	     2:23.0.0-0ubuntu1 500
		500 http://archive.ubuntu.com/ubuntu mantic/main amd64 Packages


	root@cinder-mantic:~# dpkg -l | grep cinder
	ii  cinder-api                      2:23.0.0-0ubuntu1.2                     all          Cinder storage service - API server
	ii  cinder-common                   2:23.0.0-0ubuntu1.2                     all          Cinder storage service - common files
	ii  cinder-scheduler                2:23.0.0-0ubuntu1.2                     all          Cinder storage service - Scheduler server
	ii  cinder-volume                   2:23.0.0-0ubuntu1.2                     all          Cinder storage service - Volume server
	ii  python3-cinder                  2:23.0.0-0ubuntu1.2                     all          Cinder Python 3 libraries
	ii  python3-cinderclient            1:9.4.0-0ubuntu1                        all          Python bindings to the OpenStack Volume API - Python 3.x

	root@cinder-mantic:~# systemctl show cinder-volume.service | grep Wants=
	Wants=network-online.target tgt.service

jammy:
	root@cinder-jammy:~# apt-cache policy cinder-volume
	cinder-volume:
	  Installed: 2:20.3.1-0ubuntu1.2
	  Candidate: 2:20.3.1-0ubuntu1.2
	  Version table:
	 *** 2:20.3.1-0ubuntu1.2 400
		400 http://archive.ubuntu.com/ubuntu jammy-proposed/main amd64 Packages
		100 /var/lib/dpkg/status
	     2:20.3.1-0ubuntu1.1 500
		500 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages
	     2:20.2.0-0ubuntu1.1 500
		500 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages
	     2:20.0.0-0ubuntu1 500
		500 http://archive.ubuntu.com/ubuntu jammy/main amd64 Packages

	root@cinder-jammy:~# dpkg -l | grep cinder
	ii  cinder-api                      2:20.3.1-0ubuntu1.2                     all          Cinder storage service - API server
	ii  cinder-common                   2:20.3.1-0ubuntu1.2                     all          Cinder storage service - common files
	ii  cinder-scheduler                2:20.3.1-0ubuntu1.2                     all          Cinder storage service - Scheduler server
	ii  cinder-volume                   2:20.3.1-0ubuntu1.2                     all          Cinder storage service - Volume server
	ii  python3-cinder                  2:20.3.1-0ubuntu1.2                     all          Cinder Python 3 libraries
	ii  python3-cinderclient            1:8.3.0-0ubuntu1                        all          Python bindings to the OpenStack Volume API - Python 3.x

	root@cinder-jammy:~# systemctl show cinder-volume.service | grep Wants=
	Wants=tgt.service network-online.target

focal:

	root@cinder-focal:~# apt-cache policy cinder-volume
	cinder-volume:
	  Installed: 2:16.4.2-0ubuntu2.5
	  Candidate: 2:16.4.2-0ubuntu2.5
	  Version table:
	 *** 2:16.4.2-0ubuntu2.5 500
		400 http://archive.ubuntu.com/ubuntu focal-proposed/main amd64 Packages
		100 /var/lib/dpkg/status
	     2:16.4.2-0ubuntu2.4 500
		500 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages
		500 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages
	     2:16.0.0~b3~git2020041012.eb915e2db-0ubuntu1 500
		500 http://archive.ubuntu.com/ubuntu focal/main amd64 Packages

	root@cinder-focal:~# dpkg -l | grep cinder
	ii  cinder-api                     2:16.4.2-0ubuntu2.5               all          Cinder storage service - API server
	ii  cinder-common                  2:16.4.2-0ubuntu2.5               all          Cinder storage service - common files
	ii  cinder-scheduler               2:16.4.2-0ubuntu2.5               all          Cinder storage service - Scheduler server
	ii  cinder-volume                  2:16.4.2-0ubuntu2.5               all          Cinder storage service - Volume server
	ii  python3-cinder                 2:16.4.2-0ubuntu2.5               all          Cinder Python 3 libraries
	ii  python3-cinderclient           1:7.0.0-0ubuntu1                  all          Python bindings to the OpenStack Volume API - Python 3.x

	root@cinder-focal:~# systemctl show cinder-volume.service | grep Wants=
	Wants=network-online.target tgt.service


Pre-Testing Check:

root@cinder-mantic:~# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 45b39ae0-2552-4703-a664-ae8de77a23a1 | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+

root@cinder-jammy:~# cinder list
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
| 88024eca-203c-4f8f-9ff0-4de704041a2f | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+

root@cinder-focal:~# cinder list
+--------------------------------------+--------+-------------+------+-------------+----------+-------------+
| ID                                   | Status | Name        | Size | Volume Type | Bootable | Attached to |
+--------------------------------------+--------+-------------+------+-------------+----------+-------------+
| 5fa41f17-afaf-4a56-8494-8c9246531523 | in-use | test-volume | 1    | __DEFAULT__ | false    |             |
+--------------------------------------+--------+-------------+------+-------------+----------+-------------+

Trigger the reproducer:

... Notice that cinder-volume.service is Started *AFTER* tgt.service now

... And now the 'tgtadm' error is *NOT* seen in the logs.

... And the volumes are listed with Status 'in-use' (no longer 'error')

mantic:

	root@cinder-mantic:~# date; systemctl restart cinder-volume.service tgt.service; date
	Tue Apr 30 16:34:03 UTC 2024
	Tue Apr 30 16:34:49 UTC 2024

	root@cinder-mantic:~# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -3
	Apr 30 16:34:03 cinder-mantic systemd[1]: Starting tgt.service - (i)SCSI target daemon...
	Apr 30 16:34:49 cinder-mantic systemd[1]: Started tgt.service - (i)SCSI target daemon.
	Apr 30 16:34:49 cinder-mantic systemd[1]: Started cinder-volume.service - OpenStack Cinder Volume.

	(error timestamps are from the previous/non-proposed run)
	root@cinder-mantic:~# grep 'ERROR cinder.volume.manager' /var/log/cinder/cinder-volume.log | grep tgtadm
	2024-04-30 15:54:54.603 13692 ERROR cinder.volume.manager     (out, err) = cinder.privsep.targets.tgt.tgtadm_show()
	2024-04-30 15:54:54.603 13692 ERROR cinder.volume.manager Command: tgtadm --lld iscsi --op show --mode target
	2024-04-30 15:54:54.603 13692 ERROR cinder.volume.manager Stderr: 'tgtadm: failed to send request hdr to tgt daemon, Transport endpoint is not connected\n'

	root@cinder-mantic:~# cinder list
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
	| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
	| 45b39ae0-2552-4703-a664-ae8de77a23a1 | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+


jammy:

	root@cinder-jammy:~# date; systemctl restart cinder-volume.service tgt.service; date
	Tue Apr 30 16:34:04 UTC 2024
	Tue Apr 30 16:34:50 UTC 2024

	root@cinder-jammy:~# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -3
	Apr 30 16:34:04 cinder-jammy systemd[1]: Starting (i)SCSI target daemon...
	Apr 30 16:34:50 cinder-jammy systemd[1]: Started (i)SCSI target daemon.
	Apr 30 16:34:50 cinder-jammy systemd[1]: Started OpenStack Cinder Volume.

	(error timestamps are from the previous/non-proposed run)
	root@cinder-jammy:~# grep 'ERROR cinder.volume.manager' /var/log/cinder/cinder-volume.log | grep tgtadm
	2024-04-30 15:54:55.918 14576 ERROR cinder.volume.manager     (out, err) = cinder.privsep.targets.tgt.tgtadm_show()
	2024-04-30 15:54:55.918 14576 ERROR cinder.volume.manager Command: tgtadm --lld iscsi --op show --mode target
	2024-04-30 15:54:55.918 14576 ERROR cinder.volume.manager Stderr: 'tgtadm: failed to send request hdr to tgt daemon, Transport endpoint is not connected\n'

	root@cinder-jammy:~# cinder list
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
	| ID                                   | Status | Name        | Size | Consumes Quota | Volume Type | Bootable | Attached to |
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+
	| 88024eca-203c-4f8f-9ff0-4de704041a2f | in-use | test-volume | 1    | True           | __DEFAULT__ | false    |             |
	+--------------------------------------+--------+-------------+------+----------------+-------------+----------+-------------+

focal:

	root@cinder-focal:~# date; systemctl restart cinder-volume.service tgt.service; date
	Tue Apr 30 16:34:05 UTC 2024
	Tue Apr 30 16:34:51 UTC 2024

	root@cinder-focal:~# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -3
	Apr 30 16:34:05 cinder-focal systemd[1]: Starting (i)SCSI target daemon...
	Apr 30 16:34:51 cinder-focal systemd[1]: Started (i)SCSI target daemon.
	Apr 30 16:34:51 cinder-focal systemd[1]: Started OpenStack Cinder Volume.

	(error timestamps are from the previous/non-proposed run)
	root@cinder-focal:~# grep 'ERROR cinder.volume.manager' /var/log/cinder/cinder-volume.log | grep tgtadm
	2024-04-30 15:54:55.830 15834 ERROR cinder.volume.manager     (out, err) = cinder.privsep.targets.tgt.tgtadm_show()
	2024-04-30 15:54:55.830 15834 ERROR cinder.volume.manager Command: tgtadm --lld iscsi --op show --mode target
	2024-04-30 15:54:55.830 15834 ERROR cinder.volume.manager Stderr: 'tgtadm: failed to send request hdr to tgt daemon, Transport endpoint is not connected\n'

	root@cinder-focal:~# cinder list
	+--------------------------------------+--------+-------------+------+-------------+----------+-------------+
	| ID                                   | Status | Name        | Size | Volume Type | Bootable | Attached to |
	+--------------------------------------+--------+-------------+------+-------------+----------+-------------+
	| 5fa41f17-afaf-4a56-8494-8c9246531523 | in-use | test-volume | 1    | __DEFAULT__ | false    |             |
	+--------------------------------------+--------+-------------+------+-------------+----------+-------------+

...


PLUS: in Mantic/Jammy, tgtd is not a hard dependency, thus can be removed,
and the new Wants= directive should not cause any issues.

Mantic/Jammy: without tgt.service installed:

# apt remove --purge --yes tgt

# systemctl status tgt.service
Unit tgt.service could not be found.

# systemctl show cinder-volume.service | grep Wants=
Wants=tgt.service network-online.target

# systemctl restart cinder-volume.service
# echo $?
0

root@cinder-mantic:~# systemctl status cinder-volume.service | grep Active:
     Active: active (running) since Tue 2024-04-30 17:51:20 UTC; 13s ago

root@cinder-mantic:~# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -1
Apr 30 17:51:20 cinder-mantic systemd[1]: Started cinder-volume.service - OpenStack Cinder Volume.


root@cinder-jammy:~# systemctl status cinder-volume.service | grep Active:
     Active: active (running) since Tue 2024-04-30 17:51:22 UTC; 10s ago
root@cinder-jammy:~# journalctl -b -u cinder-volume.service -u tgt.service | grep Start | tail -1
Apr 30 17:51:22 cinder-jammy systemd[1]: Started OpenStack Cinder Volume.