Another interesting observation when I am trying to use dedicated cpus, it is taking from shared cpu list only...
1. I have 4 shared and 44 dedicated cpus in the compute.
root@computedp:/home/cdot# cat /etc/nova/nova.conf cpu_shared_set = 0-3 cpu_dedicated_set = 4-47
2. I am just trying to assign cpus from dedicated list in my instance, using below flavor for that root@controller# openstack flavor show testnew +----------------------------+--------------------------------------+ | Field | Value | +----------------------------+--------------------------------------+ | OS-FLV-DISABLED:disabled | False | | OS-FLV-EXT-DATA:ephemeral | 0 | | access_project_ids | None | | description | None | | disk | 10 | | id | 37f816c6-56d7-4e16-8253-75de7d64690e | | name | testnew | | os-flavor-access:is_public | True | | properties | hw:cpu_policy='dedicated' | | ram | 8192 | | rxtx_factor | 1.0 | | swap | | | vcpus | 4 | +----------------------------+--------------------------------------+
3. nova-scheduler log is showing
2024-02-20 11:14:08.447 3247 INFO nova.virt.hardware [req-a5c01121-1224-4346-876d-d43c23b3c4b0 0adba34df2344b218b0ecd329a5464b8 f665d61d52ff4f2d963a4321b12a5842 - default default] Computed NUMA topology CPU pinning: usable pCPUs: [[2], [8], [14], [20], [26], [32], [38], [44], [4], [10], [16], [22], [28], [34], [40], [46], [0], [6], [12], [18], [24], [30], [36], [42]], vCPUs mapping: [(0, 2), (1, 8), (2, 14), (3, 20)]
4. instance is launched and taking cpus from shared list only root@computedp:# virsh vcpupin 37 VCPU CPU Affinity ---------------------- 0 0,2 1 0,2 2 0,2 3 0,2
5. instance log also doesn't contain any dedicated info root@computedp:# cat /var/log/libvirt/qemu/instance-00000085.log 2024-02-20 05:44:10.539+0000: starting up libvirt version: 6.0.0, package: 0ubuntu8.16 (Marc Deslauriers <email address hidden> Wed, 20 Apr 2022 11:31:12 -0400), qemu version: 4.2.1Debian 1:4.2-3ubuntu6.28, kernel: 5.4.0-125-generic, hostname: computedp LC_ALL=C \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin \ HOME=/var/lib/libvirt/qemu/domain-37-instance-00000085 \ XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-37-instance-00000085/.local/share \ XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-37-instance-00000085/.cache \ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-37-instance-00000085/.config \ QEMU_AUDIO_DRV=none \ /usr/bin/qemu-system-x86_64 \ -name guest=instance-00000085,debug-threads=on \ -S \ -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-37-instance-00000085/master-key.aes \ -machine pc-i440fx-4.2,accel=kvm,usb=off,dump-guest-core=off \ -cpu host \ -m 8192 \ -overcommit mem-lock=off \ -smp 4,sockets=4,cores=1,threads=1 \ -object memory-backend-ram,id=ram-node0,size=8589934592,host-nodes=0,policy=bind \ -numa node,nodeid=0,cpus=0-3,memdev=ram-node0 \ -uuid 1ae64fb2-48d6-4882-b043-73411f026c79 \ -smbios 'type=1,manufacturer=OpenStack Foundation,product=OpenStack Nova,version=23.2.1,serial=1ae64fb2-48d6-4882-b043-73411f026c79,uuid=1ae64fb2-48d6-4882-b043-73411f026c79,family=Virtual Machine' \ -no-user-config \ -nodefaults \ -chardev socket,id=charmonitor,fd=32,server,nowait \ -mon chardev=charmonitor,id=monitor,mode=control \ -rtc base=utc,driftfix=slew \ -global kvm-pit.lost_tick_policy=delay \ -no-hpet \ -no-shutdown \ -boot strict=on \ -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \ -blockdev '{"driver":"file","filename":"/var/lib/nova/instances/_base/e6b21f0cb7f233629d4fde570bccaffae2b32448","node-name":"libvirt-2-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-2-format","read-only":true,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"}' \ -blockdev '{"driver":"file","filename":"/var/lib/nova/instances/1ae64fb2-48d6-4882-b043-73411f026c79/disk","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":"libvirt-1-storage","backing":"libvirt-2-format"}' \ -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=libvirt-1-format,id=virtio-disk0,bootindex=1,write-cache=on \ -netdev tap,fd=39,id=hostnet0,vhost=on,vhostfd=40 \ -device virtio-net-pci,host_mtu=1450,netdev=hostnet0,id=net0,mac=fa:16:3e:13:60:f6,bus=pci.0,addr=0x3 \ -add-fd set=3,fd=42 \ -chardev pty,id=charserial0,logfile=/dev/fdset/3,logappend=on \ -device isa-serial,chardev=charserial0,id=serial0 \ -device usb-tablet,id=input0,bus=usb.0,port=1 \ -vnc 0.0.0.0:5 \ -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 \ -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 \ -object rng-random,id=objrng0,filename=/dev/urandom \ -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x6 \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on 2024-02-20 05:44:10.539+0000: Domain id=37 is tainted: host-cpu char device redirected to /dev/pts/19 (label charserial0)
Let me know if you find any issue with the setup or is it an expected behavior?
Another interesting observation when I am trying to use dedicated cpus, it is taking from shared cpu list only...
1. I have 4 shared and 44 dedicated cpus in the compute.
root@computedp: /home/cdot# cat /etc/nova/nova.conf
cpu_shared_set = 0-3
cpu_dedicated_set = 4-47
2. I am just trying to assign cpus from dedicated list in my instance, using below flavor for that ------- ------- ------- -+----- ------- ------- ------- ------- -----+ ------- ------- ------- -+----- ------- ------- ------- ------- -----+ DISABLED: disabled | False | EXT-DATA: ephemeral | 0 | 56d7-4e16- 8253-75de7d6469 0e | access: is_public | True | policy= 'dedicated' | ------- ------- ------- -+----- ------- ------- ------- ------- -----+
root@controller# openstack flavor show testnew
+------
| Field | Value |
+------
| OS-FLV-
| OS-FLV-
| access_project_ids | None |
| description | None |
| disk | 10 |
| id | 37f816c6-
| name | testnew |
| os-flavor-
| properties | hw:cpu_
| ram | 8192 |
| rxtx_factor | 1.0 |
| swap | |
| vcpus | 4 |
+------
3. nova-scheduler log is showing
2024-02-20 11:14:08.447 3247 INFO nova.virt.hardware [req-a5c01121- 1224-4346- 876d-d43c23b3c4 b0 0adba34df2344b2 18b0ecd329a5464 b8 f665d61d52ff4f2 d963a4321b12a58 42 - default default] Computed NUMA topology CPU pinning: usable pCPUs: [[2], [8], [14], [20], [26], [32], [38], [44], [4], [10], [16], [22], [28], [34], [40], [46], [0], [6], [12], [18], [24], [30], [36], [42]], vCPUs mapping: [(0, 2), (1, 8), (2, 14), (3, 20)]
4. instance is launched and taking cpus from shared list only ------- ------- -
root@computedp:# virsh vcpupin 37
VCPU CPU Affinity
-------
0 0,2
1 0,2
2 0,2
3 0,2
5. instance log also doesn't contain any dedicated info libvirt/ qemu/instance- 00000085. log local/sbin: /usr/local/ bin:/usr/ sbin:/usr/ bin:/sbin: /bin:/snap/ bin \ lib/libvirt/ qemu/domain- 37-instance- 00000085 \ HOME=/var/ lib/libvirt/ qemu/domain- 37-instance- 00000085/ .local/ share \ HOME=/var/ lib/libvirt/ qemu/domain- 37-instance- 00000085/ .cache \ HOME=/var/ lib/libvirt/ qemu/domain- 37-instance- 00000085/ .config \ qemu-system- x86_64 \ 00000085, debug-threads= on \ id=masterKey0, format= raw,file= /var/lib/ libvirt/ qemu/domain- 37-instance- 00000085/ master- key.aes \ 4.2,accel= kvm,usb= off,dump- guest-core= off \ 4,cores= 1,threads= 1 \ backend- ram,id= ram-node0, size=8589934592 ,host-nodes= 0,policy= bind \ 0,cpus= 0-3,memdev= ram-node0 \ 48d6-4882- b043-73411f026c 79 \ manufacturer= OpenStack Foundation, product= OpenStack Nova,version= 23.2.1, serial= 1ae64fb2- 48d6-4882- b043-73411f026c 79,uuid= 1ae64fb2- 48d6-4882- b043-73411f026c 79,family= Virtual Machine' \ id=charmonitor, fd=32,server, nowait \ charmonitor, id=monitor, mode=control \ driftfix= slew \ lost_tick_ policy= delay \ uhci,id= usb,bus= pci.0,addr= 0x1.0x2 \ :"file" ,"filename" :"/var/ lib/nova/ instances/ _base/e6b21f0cb 7f233629d4fde57 0bccaffae2b3244 8","node- name":" libvirt- 2-storage" ,"cache" :{"direct" :true," no-flush" :false} ,"auto- read-only" :true," discard" :"unmap" }' \ name":" libvirt- 2-format" ,"read- only":true, "cache" :{"direct" :true," no-flush" :false} ,"driver" :"raw", "file": "libvirt- 2-storage" }' \ :"file" ,"filename" :"/var/ lib/nova/ instances/ 1ae64fb2- 48d6-4882- b043-73411f026c 79/disk" ,"node- name":" libvirt- 1-storage" ,"cache" :{"direct" :true," no-flush" :false} ,"auto- read-only" :true," discard" :"unmap" }' \ name":" libvirt- 1-format" ,"read- only":false, "cache" :{"direct" :true," no-flush" :false} ,"driver" :"qcow2" ,"file" :"libvirt- 1-storage" ,"backing" :"libvirt- 2-format" }' \ blk-pci, scsi=off, bus=pci. 0,addr= 0x4,drive= libvirt- 1-format, id=virtio- disk0,bootindex =1,write- cache=on \ 39,id=hostnet0, vhost=on, vhostfd= 40 \ net-pci, host_mtu= 1450,netdev= hostnet0, id=net0, mac=fa: 16:3e:13: 60:f6,bus= pci.0,addr= 0x3 \ charserial0, logfile= /dev/fdset/ 3,logappend= on \ chardev= charserial0, id=serial0 \ id=input0, bus=usb. 0,port= 1 \ vga,id= video0, bus=pci. 0,addr= 0x2 \ balloon- pci,id= balloon0, bus=pci. 0,addr= 0x5 \ id=objrng0, filename= /dev/urandom \ rng-pci, rng=objrng0, id=rng0, bus=pci. 0,addr= 0x6 \ deny,elevatepri vileges= deny,spawn= deny,resourceco ntrol=deny \
root@computedp:# cat /var/log/
2024-02-20 05:44:10.539+0000: starting up libvirt version: 6.0.0, package: 0ubuntu8.16 (Marc Deslauriers <email address hidden> Wed, 20 Apr 2022 11:31:12 -0400), qemu version: 4.2.1Debian 1:4.2-3ubuntu6.28, kernel: 5.4.0-125-generic, hostname: computedp
LC_ALL=C \
PATH=/usr/
HOME=/var/
XDG_DATA_
XDG_CACHE_
XDG_CONFIG_
QEMU_AUDIO_DRV=none \
/usr/bin/
-name guest=instance-
-S \
-object secret,
-machine pc-i440fx-
-cpu host \
-m 8192 \
-overcommit mem-lock=off \
-smp 4,sockets=
-object memory-
-numa node,nodeid=
-uuid 1ae64fb2-
-smbios 'type=1,
-no-user-config \
-nodefaults \
-chardev socket,
-mon chardev=
-rtc base=utc,
-global kvm-pit.
-no-hpet \
-no-shutdown \
-boot strict=on \
-device piix3-usb-
-blockdev '{"driver"
-blockdev '{"node-
-blockdev '{"driver"
-blockdev '{"node-
-device virtio-
-netdev tap,fd=
-device virtio-
-add-fd set=3,fd=42 \
-chardev pty,id=
-device isa-serial,
-device usb-tablet,
-vnc 0.0.0.0:5 \
-device cirrus-
-device virtio-
-object rng-random,
-device virtio-
-sandbox on,obsolete=
-msg timestamp=on
2024-02-20 05:44:10.539+0000: Domain id=37 is tainted: host-cpu
char device redirected to /dev/pts/19 (label charserial0)
Let me know if you find any issue with the setup or is it an expected behavior?