Activity log for bug #1334024

Date Who What changed Old value New value Message
2014-06-24 22:01:02 Shraddha Pandhe bug added bug
2014-06-24 22:04:10 Shraddha Pandhe nova: assignee Shraddha Pandhe (shraddha-pandhe)
2014-06-24 22:17:44 Shraddha Pandhe description I am using config drive to boot VMs. In icehouse, I observed that nova rescue fails and leaves the VM in SHUTOFF state. Short error log: /home/y/var/nova/instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory Difference in Havana and Icehouse code path: # Havana # Config drive if configdrive.required_by(instance): LOG.info(_('Using config drive'), instance=instance) extra_md = {} if admin_pass: extra_md['admin_pass'] = admin_pass for f in ('user_name', 'project_name'): if hasattr(context, f): extra_md[f] = getattr(context, f, None) inst_md = instance_metadata.InstanceMetadata(instance, content=files, extra_md=extra_md, network_info=network_info) with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb: configdrive_path = basepath(fname='disk.config') LOG.info(_('Creating config drive at %(path)s'), {'path': configdrive_path}, instance=instance) def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config. return os.path.join(libvirt_utils.get_instance_path(instance), fname + suffix) # Icehouse: # Config drive if configdrive.required_by(instance): LOG.info(_('Using config drive'), instance=instance) extra_md = {} if admin_pass: extra_md['admin_pass'] = admin_pass for f in ('user_name', 'project_name'): if hasattr(context, f): extra_md[f] = getattr(context, f, None) inst_md = instance_metadata.InstanceMetadata(instance, content=files, extra_md=extra_md, network_info=network_info) with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb: configdrive_path = self._get_disk_config_path(instance) LOG.info(_('Creating config drive at %(path)s'), {'path': configdrive_path}, instance=instance) @staticmethod def _get_disk_config_path(instance): return os.path.join(libvirt_utils.get_instance_path(instance), 'disk.config') The suffix .rescue is missed here and hence, original disk.config is overwritten. Following change fixed the issue for me: configdrive_path = self._get_disk_config_path(instance, suffix) @staticmethod def _get_disk_config_path(instance, suffix=''): return os.path.join(libvirt_utils.get_instance_path(instance), 'disk.config' + suffix) Complete Error log: {"extra": {"project_name": "admin", "timestamp": "2014-06-20T23:00:50.269421", "auth_token": "17fcde000c3040f9981e1804cdaf94fe", "remote_address": "10.220.4.45", "quota_class": null, "is_admin": true, "user": "dfac8c9e704a4312b0447b26b57a12da", "service_catalog": [], "tenant": "13b05759646b41c9a51660a1e653b146", "user_id": "dfac8c9e704a4312b0447b26b57a12da", "roles": ["admin"], "project": "nova.compute.manager", "instance": "[instance: 270e299b-90b2-46d5-bf9a-e7f6efe3742e] ", "version": "unknown", "read_deleted": "no", "request_id": "req-a7f12cd9-dd23-48fa-9479-ab4e7825ae1e", "instance_lock_checked": false, "project_id": "13b05759646b41c9a51660a1e653b146", "user_name": "spandhe"}, "thread_name": "GreenThread-31", "process_name": "MainProcess", "name": "nova.compute.manager", "thread": 82908816, "created": 1403305252.7287409, "process": 26178, "relative_created": 114349619.65203285, "args": [], "traceback": ["Traceback (most recent call last):", " File \"/usr/lib/python2.6/site-packages/nova/compute/manager.py\", line 2983, in rescue_instance", " rescue_image_meta, admin_password)", " File \"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 2205, in rescue", " self._create_domain(xml)", " File \"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 3562, in _create_domain", " domain.XMLDesc(0))", " File \"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py\", line 68, in __exit__", " six.reraise(self.type_, self.value, self.tb)", " File \"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 3557, in _create_domain", " domain.createWithFlags(launch_flags)", " File \"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 179, in doit", " result = proxy_call(self._autowrap, f, *args, **kwargs)", " File \"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 139, in proxy_call", " rv = execute(f,*args,**kwargs)", " File \"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 77, in tworker", " rv = meth(*args,**kwargs)", " File \"/usr/lib64/python2.6/site-packages/libvirt.py\", line 708, in createWithFlags", " if ret == -1: raise libvirtError ('virDomainCreateWithFlags() failed', dom=self)", "libvirtError: cannot open file '/home/y/var/nova/instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory"], "module": "manager", "filename": "manager.py", "levelno": 40, "msecs": 728.74093055725098, "pathname": "/usr/lib/python2.6/site-packages/nova/compute/manager.py", "lineno": 2986, "asctime": "2014-06-20 23:00:52,728", "msg": "Error trying to Rescue Instance", "message": "Error trying to Rescue Instance", "funcname": "rescue_instance", "levelname": "ERROR"} I am using config drive to boot VMs. In icehouse, I observed that nova rescue fails and leaves the VM in SHUTOFF state. Short error log: instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory Difference in Havana and Icehouse code path: # Havana # Config drive         if configdrive.required_by(instance):             LOG.info(_('Using config drive'), instance=instance)             extra_md = {}             if admin_pass:                 extra_md['admin_pass'] = admin_pass             for f in ('user_name', 'project_name'):                 if hasattr(context, f):                     extra_md[f] = getattr(context, f, None)             inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)             with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:                 configdrive_path = basepath(fname='disk.config')                 LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.     return os.path.join(libvirt_utils.get_instance_path(instance),                                 fname + suffix) # Icehouse: # Config drive         if configdrive.required_by(instance):             LOG.info(_('Using config drive'), instance=instance)             extra_md = {}             if admin_pass:                 extra_md['admin_pass'] = admin_pass             for f in ('user_name', 'project_name'):                 if hasattr(context, f):                     extra_md[f] = getattr(context, f, None)             inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)             with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:                 configdrive_path = self._get_disk_config_path(instance)                 LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) @staticmethod def _get_disk_config_path(instance):     return os.path.join(libvirt_utils.get_instance_path(instance),                         'disk.config') The suffix .rescue is missed here and hence, original disk.config is overwritten. Following change fixed the issue for me: configdrive_path = self._get_disk_config_path(instance, suffix) @staticmethod def _get_disk_config_path(instance, suffix=''):     return os.path.join(libvirt_utils.get_instance_path(instance),                             'disk.config' + suffix)
2014-06-24 22:24:52 Shraddha Pandhe description I am using config drive to boot VMs. In icehouse, I observed that nova rescue fails and leaves the VM in SHUTOFF state. Short error log: instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory Difference in Havana and Icehouse code path: # Havana # Config drive         if configdrive.required_by(instance):             LOG.info(_('Using config drive'), instance=instance)             extra_md = {}             if admin_pass:                 extra_md['admin_pass'] = admin_pass             for f in ('user_name', 'project_name'):                 if hasattr(context, f):                     extra_md[f] = getattr(context, f, None)             inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)             with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:                 configdrive_path = basepath(fname='disk.config')                 LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.     return os.path.join(libvirt_utils.get_instance_path(instance),                                 fname + suffix) # Icehouse: # Config drive         if configdrive.required_by(instance):             LOG.info(_('Using config drive'), instance=instance)             extra_md = {}             if admin_pass:                 extra_md['admin_pass'] = admin_pass             for f in ('user_name', 'project_name'):                 if hasattr(context, f):                     extra_md[f] = getattr(context, f, None)             inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)             with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:                 configdrive_path = self._get_disk_config_path(instance)                 LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) @staticmethod def _get_disk_config_path(instance):     return os.path.join(libvirt_utils.get_instance_path(instance),                         'disk.config') The suffix .rescue is missed here and hence, original disk.config is overwritten. Following change fixed the issue for me: configdrive_path = self._get_disk_config_path(instance, suffix) @staticmethod def _get_disk_config_path(instance, suffix=''):     return os.path.join(libvirt_utils.get_instance_path(instance),                             'disk.config' + suffix) I am using config drive to boot VMs. In icehouse, I observed that nova rescue fails and leaves the VM in SHUTOFF state. Short error log: instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory Difference in Havana and Icehouse code path: # Havana # Config drive  if configdrive.required_by(instance):      LOG.info(_('Using config drive'), instance=instance)      extra_md = {}      if admin_pass:       extra_md['admin_pass'] = admin_pass      for f in ('user_name', 'project_name'):      if hasattr(context, f):              extra_md[f] = getattr(context, f, None)          inst_md = instance_metadata.InstanceMetadata(instance,                content=files, extra_md=extra_md, network_info=network_info)          with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:           configdrive_path = basepath(fname='disk.config')              LOG.info(_('Creating config drive at %(path)s'),                       {'path': configdrive_path}, instance=instance) def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.     return os.path.join(libvirt_utils.get_instance_path(instance),                                 fname + suffix) # Icehouse: # Config drive if configdrive.required_by(instance):     LOG.info(_('Using config drive'), instance=instance)     extra_md = {}     if admin_pass:     extra_md['admin_pass'] = admin_pass     for f in ('user_name', 'project_name'):         if hasattr(context, f):             extra_md[f] = getattr(context, f, None)         inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)         with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:          configdrive_path = self._get_disk_config_path(instance)             LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) @staticmethod def _get_disk_config_path(instance):     return os.path.join(libvirt_utils.get_instance_path(instance),                         'disk.config') The suffix .rescue is missed here and hence, original disk.config is overwritten. Following change fixed the issue for me: configdrive_path = self._get_disk_config_path(instance, suffix) @staticmethod def _get_disk_config_path(instance, suffix=''):     return os.path.join(libvirt_utils.get_instance_path(instance),                             'disk.config' + suffix)
2014-06-24 22:25:28 Shraddha Pandhe description I am using config drive to boot VMs. In icehouse, I observed that nova rescue fails and leaves the VM in SHUTOFF state. Short error log: instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory Difference in Havana and Icehouse code path: # Havana # Config drive  if configdrive.required_by(instance):      LOG.info(_('Using config drive'), instance=instance)      extra_md = {}      if admin_pass:       extra_md['admin_pass'] = admin_pass      for f in ('user_name', 'project_name'):      if hasattr(context, f):              extra_md[f] = getattr(context, f, None)          inst_md = instance_metadata.InstanceMetadata(instance,                content=files, extra_md=extra_md, network_info=network_info)          with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:           configdrive_path = basepath(fname='disk.config')              LOG.info(_('Creating config drive at %(path)s'),                       {'path': configdrive_path}, instance=instance) def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.     return os.path.join(libvirt_utils.get_instance_path(instance),                                 fname + suffix) # Icehouse: # Config drive if configdrive.required_by(instance):     LOG.info(_('Using config drive'), instance=instance)     extra_md = {}     if admin_pass:     extra_md['admin_pass'] = admin_pass     for f in ('user_name', 'project_name'):         if hasattr(context, f):             extra_md[f] = getattr(context, f, None)         inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)         with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:          configdrive_path = self._get_disk_config_path(instance)             LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) @staticmethod def _get_disk_config_path(instance):     return os.path.join(libvirt_utils.get_instance_path(instance),                         'disk.config') The suffix .rescue is missed here and hence, original disk.config is overwritten. Following change fixed the issue for me: configdrive_path = self._get_disk_config_path(instance, suffix) @staticmethod def _get_disk_config_path(instance, suffix=''):     return os.path.join(libvirt_utils.get_instance_path(instance),                             'disk.config' + suffix) I am using config drive to boot VMs. In icehouse, I observed that nova rescue fails and leaves the VM in SHUTOFF state. Short error log: instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory Difference in Havana and Icehouse code path: # Havana # Config drive  if configdrive.required_by(instance):      LOG.info(_('Using config drive'), instance=instance)      extra_md = {}      if admin_pass:       extra_md['admin_pass'] = admin_pass      for f in ('user_name', 'project_name'):          if hasattr(context, f):           extra_md[f] = getattr(context, f, None)          inst_md = instance_metadata.InstanceMetadata(instance,                content=files, extra_md=extra_md, network_info=network_info)          with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:           configdrive_path = basepath(fname='disk.config')              LOG.info(_('Creating config drive at %(path)s'),                       {'path': configdrive_path}, instance=instance) def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.     return os.path.join(libvirt_utils.get_instance_path(instance),                                 fname + suffix) # Icehouse: # Config drive if configdrive.required_by(instance):     LOG.info(_('Using config drive'), instance=instance)     extra_md = {}     if admin_pass:         extra_md['admin_pass'] = admin_pass     for f in ('user_name', 'project_name'):         if hasattr(context, f):             extra_md[f] = getattr(context, f, None)         inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)         with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:          configdrive_path = self._get_disk_config_path(instance)             LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) @staticmethod def _get_disk_config_path(instance):     return os.path.join(libvirt_utils.get_instance_path(instance),                         'disk.config') The suffix .rescue is missed here and hence, original disk.config is overwritten. Following change fixed the issue for me: configdrive_path = self._get_disk_config_path(instance, suffix) @staticmethod def _get_disk_config_path(instance, suffix=''):     return os.path.join(libvirt_utils.get_instance_path(instance),                             'disk.config' + suffix)
2014-06-24 22:27:55 Shraddha Pandhe description I am using config drive to boot VMs. In icehouse, I observed that nova rescue fails and leaves the VM in SHUTOFF state. Short error log: instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory Difference in Havana and Icehouse code path: # Havana # Config drive  if configdrive.required_by(instance):      LOG.info(_('Using config drive'), instance=instance)      extra_md = {}      if admin_pass:       extra_md['admin_pass'] = admin_pass      for f in ('user_name', 'project_name'):          if hasattr(context, f):           extra_md[f] = getattr(context, f, None)          inst_md = instance_metadata.InstanceMetadata(instance,                content=files, extra_md=extra_md, network_info=network_info)          with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:           configdrive_path = basepath(fname='disk.config')              LOG.info(_('Creating config drive at %(path)s'),                       {'path': configdrive_path}, instance=instance) def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.     return os.path.join(libvirt_utils.get_instance_path(instance),                                 fname + suffix) # Icehouse: # Config drive if configdrive.required_by(instance):     LOG.info(_('Using config drive'), instance=instance)     extra_md = {}     if admin_pass:         extra_md['admin_pass'] = admin_pass     for f in ('user_name', 'project_name'):         if hasattr(context, f):             extra_md[f] = getattr(context, f, None)         inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)         with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:          configdrive_path = self._get_disk_config_path(instance)             LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) @staticmethod def _get_disk_config_path(instance):     return os.path.join(libvirt_utils.get_instance_path(instance),                         'disk.config') The suffix .rescue is missed here and hence, original disk.config is overwritten. Following change fixed the issue for me: configdrive_path = self._get_disk_config_path(instance, suffix) @staticmethod def _get_disk_config_path(instance, suffix=''):     return os.path.join(libvirt_utils.get_instance_path(instance),                             'disk.config' + suffix) I am using config drive to boot VMs. In icehouse, I observed that nova rescue fails and leaves the VM in SHUTOFF state. Short error log: instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory Difference in Havana and Icehouse code path: # Havana # Config drive  if configdrive.required_by(instance):      LOG.info(_('Using config drive'), instance=instance)      extra_md = {}       if admin_pass:       extra_md['admin_pass'] = admin_pass       for f in ('user_name', 'project_name'):          if hasattr(context, f):           extra_md[f] = getattr(context, f, None)          inst_md = instance_metadata.InstanceMetadata(instance,                content=files, extra_md=extra_md, network_info=network_info)          with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:           configdrive_path = basepath(fname='disk.config')              LOG.info(_('Creating config drive at %(path)s'),                       {'path': configdrive_path}, instance=instance) def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.     return os.path.join(libvirt_utils.get_instance_path(instance),                                 fname + suffix) # Icehouse: # Config drive if configdrive.required_by(instance):     LOG.info(_('Using config drive'), instance=instance)     extra_md = {}     if admin_pass:         extra_md['admin_pass'] = admin_pass     for f in ('user_name', 'project_name'):         if hasattr(context, f):             extra_md[f] = getattr(context, f, None)         inst_md = instance_metadata.InstanceMetadata(instance,                 content=files, extra_md=extra_md, network_info=network_info)         with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:          configdrive_path = self._get_disk_config_path(instance)             LOG.info(_('Creating config drive at %(path)s'),                          {'path': configdrive_path}, instance=instance) @staticmethod def _get_disk_config_path(instance):     return os.path.join(libvirt_utils.get_instance_path(instance),                         'disk.config') The suffix .rescue is missed here and hence, original disk.config is overwritten. Following change fixed the issue for me: configdrive_path = self._get_disk_config_path(instance, suffix) @staticmethod def _get_disk_config_path(instance, suffix=''):     return os.path.join(libvirt_utils.get_instance_path(instance),                             'disk.config' + suffix)
2014-06-24 22:54:39 melanie witt nova: importance Undecided Medium
2014-06-24 22:54:39 melanie witt nova: status New Confirmed
2014-06-24 22:56:25 melanie witt tags libvirt
2014-06-24 22:57:03 Suneel K Bomminayuni bug added subscriber Suneel K Bomminayuni
2014-06-25 21:03:08 OpenStack Infra nova: status Confirmed In Progress
2014-06-26 02:37:25 Michael Still tags libvirt icehouse-backport-potential libvirt
2014-07-22 07:25:56 OpenStack Infra nova: status In Progress Fix Committed
2014-07-23 14:58:09 Russell Bryant nova: status Fix Committed Fix Released
2014-07-23 14:58:09 Russell Bryant nova: milestone juno-2
2014-10-16 08:39:35 Thierry Carrez nova: milestone juno-2 2014.2