resizing from flavor with swap to one without swap puts instance into Error status

Bug #1552777 reported by Chris Friesen on 2016-03-03
30
This bug affects 5 people
Affects Status Importance Assigned to Milestone
OpenStack Compute (nova)
Low
Unassigned

Bug Description

In a single-node devstack (current trunk, nova commit 6e1051b7), if you boot an instance with a flavor that has nonzero swap and then resize to a flavor with zero swap it causes an exception. It seems that we somehow neglect to remove the swap file from the instance.

 2016-03-03 10:02:41.415 ERROR nova.virt.libvirt.guest [req-dadee404-81c4-46de-9fd5-58de747b3b78 admin alt_demo] Error launching a defined domain with XML: <domain type='kvm'>
  <name>instance-00000001</name>
  <uuid>54711b56-fa72-4eac-a5d3-aa29ed128098</uuid>
  <metadata>
    <nova:instance xmlns:nova="http://openstack.org/xmlns/libvirt/nova/1.0">
      <nova:package version="13.0.0"/>
      <nova:name>asdf</nova:name>
      <nova:creationTime>2016-03-03 16:02:39</nova:creationTime>
      <nova:flavor name="m1.tiny">
        <nova:memory>512</nova:memory>
        <nova:disk>1</nova:disk>
        <nova:swap>0</nova:swap>
        <nova:ephemeral>0</nova:ephemeral>
        <nova:vcpus>1</nova:vcpus>
      </nova:flavor>
      <nova:owner>
        <nova:user uuid="96e62e9067124d79aefb0234143287eb">admin</nova:user>
        <nova:project uuid="376f545938224be4ae76f283144dc31d">alt_demo</nova:project>
      </nova:owner>
      <nova:root type="image" uuid="d30c2fce-f882-4720-8367-e416cc15a32e"/>
    </nova:instance>
  </metadata>
  <memory unit='KiB'>524288</memory>
  <currentMemory unit='KiB'>524288</currentMemory>
  <vcpu placement='static'>1</vcpu>
  <cputune>
    <shares>1024</shares>
  </cputune>
  <sysinfo type='smbios'>
    <system>
      <entry name='manufacturer'>OpenStack Foundation</entry>
      <entry name='product'>OpenStack Nova</entry>
      <entry name='version'>13.0.0</entry>
      <entry name='serial'>03000200-0400-0500-0006-000700080009</entry>
      <entry name='uuid'>54711b56-fa72-4eac-a5d3-aa29ed128098</entry>
      <entry name='family'>Virtual Machine</entry>
    </system>
  </sysinfo>
  <os>
    <type arch='x86_64' machine='pc-i440fx-utopic'>hvm</type>
    <kernel>/opt/stack/data/nova/instances/54711b56-fa72-4eac-a5d3-aa29ed128098/kernel</kernel>
    <initrd>/opt/stack/data/nova/instances/54711b56-fa72-4eac-a5d3-aa29ed128098/ramdisk</initrd>
    <cmdline>root=/dev/vda console=tty0 console=ttyS0</cmdline>
    <boot dev='hd'/>
    <smbios mode='sysinfo'/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <cpu>
    <topology sockets='1' cores='1' threads='1'/>
  </cpu>
  <clock offset='utc'>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='hpet' present='no'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/bin/kvm-spice</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2' cache='none'/>
      <source file='/opt/stack/data/nova/instances/54711b56-fa72-4eac-a5d3-aa29ed128098/disk'/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2' cache='none'/>
      <source file='/opt/stack/data/nova/instances/54711b56-fa72-4eac-a5d3-aa29ed128098/disk.swap'/>
      <target dev='vdb' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </disk>
    <disk type='file' device='cdrom'>
      <driver name='qemu' type='raw' cache='none'/>
      <source file='/opt/stack/data/nova/instances/54711b56-fa72-4eac-a5d3-aa29ed128098/disk.config'/>
      <target dev='hdd' bus='ide'/>
      <readonly/>
      <address type='drive' controller='0' bus='1' target='0' unit='1'/>
    </disk>
    <controller type='usb' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'/>
    <controller type='ide' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
    </controller>
    <interface type='bridge'>
      <mac address='fa:16:3e:a9:b5:c5'/>
      <source bridge='br100'/>
      <model type='virtio'/>
      <filterref filter='nova-instance-instance-00000001-fa163ea9b5c5'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='file'>
      <source path='/opt/stack/data/nova/instances/54711b56-fa72-4eac-a5d3-aa29ed128098/console.log'/>
      <target port='0'/>
    </serial>
    <serial type='pty'>
      <target port='1'/>
    </serial>
    <console type='file'>
      <source path='/opt/stack/data/nova/instances/54711b56-fa72-4eac-a5d3-aa29ed128098/console.log'/>
      <target type='serial' port='0'/>
    </console>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1' keymap='en-us'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <video>
      <model type='cirrus' vram='16384' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
      <stats period='10'/>
    </memballoon>
  </devices>
</domain>

2016-03-03 10:02:41.417 ERROR nova.compute.manager [req-dadee404-81c4-46de-9fd5-58de747b3b78 admin alt_demo] [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] Setting instance vm_state to ERROR
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] Traceback (most recent call last):
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/opt/stack/nova/nova/compute/manager.py", line 3999, in finish_resize
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] disk_info, image_meta)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/opt/stack/nova/nova/compute/manager.py", line 3964, in _finish_resize
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] old_instance_type)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 204, in __exit__
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] six.reraise(self.type_, self.value, self.tb)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/opt/stack/nova/nova/compute/manager.py", line 3959, in _finish_resize
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] block_device_info, power_on)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 7202, in finish_migration
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] vifs_already_plugged=True)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 4862, in _create_domain_and_network
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] xml, pause=pause, power_on=power_on)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 4793, in _create_domain
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] guest.launch(pause=pause)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/opt/stack/nova/nova/virt/libvirt/guest.py", line 142, in launch
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] self._encoded_xml, errors='ignore')
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 204, in __exit__
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] six.reraise(self.type_, self.value, self.tb)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/opt/stack/nova/nova/virt/libvirt/guest.py", line 137, in launch
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] return self._domain.createWithFlags(flags)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 183, in doit
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] result = proxy_call(self._autowrap, f, *args, **kwargs)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 141, in proxy_call
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] rv = execute(f, *args, **kwargs)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 122, in execute
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] six.reraise(c, e, tb)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/usr/local/lib/python2.7/dist-packages/eventlet/tpool.py", line 80, in tworker
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] rv = meth(*args, **kwargs)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] File "/usr/local/lib/python2.7/dist-packages/libvirt.py", line 1037, in createWithFlags
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] if ret == -1: raise libvirtError ('virDomainCreateWithFlags() failed', dom=self)
2016-03-03 10:02:41.417 TRACE nova.compute.manager [instance: 54711b56-fa72-4eac-a5d3-aa29ed128098] libvirtError: Cannot access storage file '/opt/stack/data/nova/instances/54711b56-fa72-4eac-a5d3-aa29ed128098/disk.swap' (as uid:109, gid:118): No such file or directory

Chris Friesen (cbf123) on 2016-03-03
summary: - resizing from flavor with swap to one without breaks
+ resizing from flavor with swap to one without swap raises exception
summary: - resizing from flavor with swap to one without swap raises exception
+ resizing from flavor with swap to one without swap puts instance into
+ ERROR state
summary: resizing from flavor with swap to one without swap puts instance into
- ERROR state
+ Error status
Matt Riedemann (mriedem) on 2016-03-03
tags: added: libvirt volumes
removed: compute
Changed in nova:
status: New → Confirmed
importance: Undecided → Medium
Rajesh Tailor (ratailor) on 2016-03-04
Changed in nova:
assignee: nobody → Rajesh Tailor (ratailor)
Rajesh Tailor (ratailor) on 2016-03-28
Changed in nova:
assignee: Rajesh Tailor (ratailor) → nobody
Kam Nasim (knasim-wrs) on 2016-03-28
Changed in nova:
assignee: nobody → Kam Nasim (knasim-wrs)
Kam Nasim (knasim-wrs) on 2016-03-28
Changed in nova:
assignee: Kam Nasim (knasim-wrs) → nobody
assignee: nobody → Kam Nasim (knasim-wrs)
Kam Nasim (knasim-wrs) on 2016-03-29
Changed in nova:
status: Confirmed → In Progress
Revision history for this message
OpenStack Infra (hudson-openstack) wrote : Fix proposed to nova (master)

Fix proposed to branch: master
Review: https://review.openstack.org/300616

Revision history for this message
Chris Friesen (cbf123) wrote :

For whatever reason, it appears that this is no longer an issue as of nova commit 26117cf5 (in Pike). I was able to successfully resize from an instance with 128MB of swap to an instance with zero swap.

Chris Friesen (cbf123) on 2018-03-21
Changed in nova:
status: In Progress → Fix Released
Revision history for this message
OpenStack Infra (hudson-openstack) wrote : Change abandoned on nova (master)

Change abandoned by Matt Riedemann (<email address hidden>) on branch: master
Review: https://review.opendev.org/300616
Reason: Is this still a problem that anyone is working on? Can it be recreated in Stein? This has been sitting with -1s for over a year so I'm going to abandon it. I would suggest trying to get Lee Yarwood (lyarwood) to take a look at this.

Revision history for this message
Matt Riedemann (mriedem) wrote :

Regarding comment #2, someone said they still have the problem in Queens (maybe using a different libvirt / qemu version if that matters?):

Maxence Sartiaux
Apr 9, 2018

Patch Set 6:

I tried in Queens, still have the error, libvirt try to open the swap file

Changed in nova:
assignee: Kam Nasim (knasim-wrs) → nobody
status: Fix Released → New
importance: Medium → Undecided
Revision history for this message
Balazs Gibizer (balazs-gibizer) wrote :

I can still reproduce the problem on nova master: http://paste.openstack.org/show/779432/

Changed in nova:
status: New → Confirmed
Revision history for this message
Laurent Dumont (baconpackets) wrote :

I seem to be seeing the same issue in Queens RH OSP.

Revision history for this message
Artom Lifshitz (notartom) wrote :

Also reported by OSP customers in 13/queens: https://bugzilla.redhat.com/show_bug.cgi?id=1774332 (although unfortunately that BZ is private)

Revision history for this message
Wendy Mitchell (wmitchellwr) wrote :
Download full text (4.0 KiB)

2020-04-22_00-10-00
Instance fault (500 LibvirtError) on resize operation (cannot access disk.swap storage file)
Can still be reproduced on resize where flavors have swap

{u'message': u'libvirtError', u'code': 500, u'details': u'Traceback (most recent call last):\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/compute/manager.py", line 207, in decorated_function\n return function(self, context, *args, **kwargs)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/compute/manager.py", line 5159, in finish_resize\n context, instance, migration)\n File "/var/lib/openstack/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in _exit\n self.force_reraise()\n File "/var/lib/openstack/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n six.reraise(self.type, self.value, self.tb)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/compute/manager.py", line 5141, in finish_resize\n migration, request_spec)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/compute/manager.py", line 5175, in finish_resize_helper\n request_spec)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/compute/manager.py", line 5111, in _finish_resize\n self._set_instance_info(instance, old_flavor)\n File "/var/lib/openstack/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit\n self.force_reraise()\n File "/var/lib/openstack/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n six.reraise(self.type, self.value, self.tb)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/compute/manager.py", line 5099, in finish_resize\n block_device_info, power_on)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 9896, in finish_migration\n post_xml_callback=gen_confdrive)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 6262, in _create_domain_and_network\n destroy_disks_on_failure)\n File "/var/lib/openstack/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit\n self.force_reraise()\n File "/var/lib/openstack/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n six.reraise(self.type, self.value, self.tb)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 6231, in create_domain_and_network\n post_xml_callback=post_xml_callback)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 6165, in _create_domain\n guest.launch(pause=pause)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/virt/libvirt/guest.py", line 143, in launch\n self._encoded_xml, errors=\'ignore\')\n File "/var/lib/openstack/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit\n self.force_reraise()\n File "/var/lib/openstack/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n six.reraise(self.type, self.value, self.tb)\n File "/var/lib/openstack/lib/python2.7/site-packages/nova/virt/libvirt/guest.py", line 138, in launch\n return self._domain.createWithFlags(flags)\n File "/var/lib/openstack/lib/python2.7/site-packages/ev...

Read more...

Changed in nova:
importance: Undecided → Low
To post a comment you must log in.
This report contains Public information  Edit
Everyone can see this information.

Duplicates of this bug

Other bug subscribers

Remote bug watches

Bug watches keep track of this bug in other bug trackers.