Here is a better section of the log.



2014-10-15 12:11:47.935 11918 DEBUG oslo.messaging._drivers.amqpdriver [-] received {u'_msg_id': u'12b3321045f94363b0db0db6817795d1', u'args': {u'node_obj': {u'ironic_object.namespace': u'ironic', u'ironic_object.data': {u'instance_uuid': u'a02787c4-b446-4ede-88f7-d5fc45e18660', u'target_power_state': None, u'instance_info': {u'ramdisk': u'982dc3dd-270c-4b34-ad61-8c2416191fa8', u'kernel': u'd2d7e186-dee7-4147-83a8-fff4267c309a', u'root_gb': u'30', u'image_source': u'b588925c-e866-404e-9de9-319db44520a7', u'ephemeral_format': u'ext4', u'ephemeral_gb': u'1770', u'preserve_ephemeral': u'True', u'deploy_key': u'9Z2H8Q3F2LW3O7MOOKEOJTZ1HA2ITEC0', u'swap_mb': u'0'}, u'uuid': u'6adf9e06-8b0f-4aa9-81ca-d036a15ae45b', u'driver_info': {u'pxe_deploy_ramdisk': u'baff7d11-e69c-49ff-bd26-2c22e0585564', u'pxe_deploy_kernel': u'17e83e20-e402-4f84-8980-d87efb8259c7', u'ipmi_address': u'10.22.28.109', u'ipmi_username': u'admin', u'ipmi_password': '<SANITIZED>'}, u'target_provision_state': None, u'last_error': u"Failed to deploy. Error: Unexpected error while running command.\nCommand: qemu-img convert -O raw /mnt/state/var/lib/ironic/master_images/tmphsmp3Z/b588925c-e866-404e-9de9-319db44520a7.part /mnt/state/var/lib/ironic/master_images/tmphsmp3Z/b588925c-e866-404e-9de9-319db44520a7.converted\nExit code: 1\nStdout: ''\nStderr: 'qemu-img: error while writing sector 2768741: No space left on device\\n'", u'console_enabled': False, u'extra': {}, u'driver': u'pxe_ipmitool', u'updated_at': u'2014-10-15T12:11:34Z', u'chassis_id': None, u'id': 1, u'provision_updated_at': u'2014-10-15T12:00:45.000000', u'maintenance': False, u'provision_state': u'deploy failed', u'reservation': None, u'created_at': u'2014-10-09T14:20:27Z', u'power_state': u'power on', u'properties': {u'memory_mb': u'128000', u'cpu_arch': u'amd64', u'local_gb': u'1800', u'cpus': u'32'}}, u'ironic_object.version': u'1.3', u'ironic_object.name': u'Node'}}, u'version': u'1.1', u'_context_request_id': u'req-23e9b3e8-e3e9-4452-a7bd-bca5f98fdddc', u'_unique_id': u'debb181fed27493197776f5e8a7f7e73', u'_reply_q': u'reply_2fb29307fa984431aea2bd907d40ef6a', u'_context_domain_id': u'default', u'_context_tenant': u'service', u'_context_is_public_api': False, u'_context_auth_token': '<SANITIZED>', u'_context_show_deleted': False, u'_context_domain_name': u'Default', u'_context_read_only': False, u'_context_user': u'ironic', u'method': u'update_node', u'_context_is_admin': True} _safe_log /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/oslo/messaging/_drivers/common.py:180
2014-10-15 12:11:47.936 11918 DEBUG oslo.messaging._drivers.amqp [-] unpacked context: {u'read_only': False, u'show_deleted': False, u'auth_token': '<SANITIZED>', u'domain_name': u'Default', u'is_admin': True, u'user': u'ironic', u'request_id': u'req-23e9b3e8-e3e9-4452-a7bd-bca5f98fdddc', u'is_public_api': False, u'domain_id': u'default', u'tenant': u'service'} _safe_log /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/oslo/messaging/_drivers/common.py:180
2014-10-15 12:11:47.940 11918 DEBUG ironic.conductor.manager [-] RPC update_node called for node 6adf9e06-8b0f-4aa9-81ca-d036a15ae45b. update_node /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/conductor/manager.py:228
2014-10-15 12:11:47.940 11918 DEBUG ironic.conductor.task_manager [-] Attempting to reserve node 6adf9e06-8b0f-4aa9-81ca-d036a15ae45b reserve_node /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/conductor/task_manager.py:182
2014-10-15 12:11:47.953 11918 DEBUG oslo.messaging._drivers.amqp [-] UNIQUE_ID is 3fa5690a0c6a4b7aafbe460716dfa24a. _add_unique_id /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/oslo/messaging/_drivers/amqp.py:249
2014-10-15 12:11:47.955 11918 DEBUG oslo.messaging._drivers.amqp [-] UNIQUE_ID is 21d9c9626b0f461da08de413c10fcd79. _add_unique_id /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/oslo/messaging/_drivers/amqp.py:249
2014-10-15 12:11:48.155 11918 DEBUG oslo.messaging._drivers.amqpdriver [-] received {u'_msg_id': u'4d30fb06534244b68f58b1b7cd8b51d9', u'args': {u'rebuild': True, u'node_id': u'6adf9e06-8b0f-4aa9-81ca-d036a15ae45b'}, u'version': u'1.15', u'_context_request_id': u'req-82d204cd-5da4-4a80-9587-93432d404805', u'_unique_id': u'8c7ba72da8014a80ac965aaa4de733c6', u'_reply_q': u'reply_2fb29307fa984431aea2bd907d40ef6a', u'_context_domain_id': u'default', u'_context_tenant': u'service', u'_context_is_public_api': False, u'_context_auth_token': '<SANITIZED>', u'_context_show_deleted': False, u'_context_domain_name': u'Default', u'_context_read_only': False, u'_context_user': u'ironic', u'method': u'do_node_deploy', u'_context_is_admin': True} _safe_log /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/oslo/messaging/_drivers/common.py:180
2014-10-15 12:11:48.155 11918 DEBUG oslo.messaging._drivers.amqp [-] unpacked context: {u'read_only': False, u'show_deleted': False, u'auth_token': '<SANITIZED>', u'domain_name': u'Default', u'is_admin': True, u'user': u'ironic', u'request_id': u'req-82d204cd-5da4-4a80-9587-93432d404805', u'is_public_api': False, u'domain_id': u'default', u'tenant': u'service'} _safe_log /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/oslo/messaging/_drivers/common.py:180
2014-10-15 12:11:48.156 11918 DEBUG ironic.conductor.manager [-] RPC do_node_deploy called for node 6adf9e06-8b0f-4aa9-81ca-d036a15ae45b. do_node_deploy /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/conductor/manager.py:408
2014-10-15 12:11:48.156 11918 DEBUG ironic.conductor.task_manager [-] Attempting to reserve node 6adf9e06-8b0f-4aa9-81ca-d036a15ae45b reserve_node /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/conductor/task_manager.py:182
2014-10-15 12:11:48.322 11918 DEBUG ironic.common.glance_service.base_image_service [-] Getting image metadata from glance. Image: b588925c-e866-404e-9de9-319db44520a7 _show /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/glance_service/base_image_service.py:185
2014-10-15 12:11:48.451 11918 DEBUG oslo.messaging._drivers.amqp [-] UNIQUE_ID is f9d5315c28d64ea59d6304e4180e29fd. _add_unique_id /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/oslo/messaging/_drivers/amqp.py:249
2014-10-15 12:11:48.453 11918 DEBUG oslo.messaging._drivers.amqp [-] UNIQUE_ID is c29325b751d6408b9609402fec9db71d. _add_unique_id /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/oslo/messaging/_drivers/amqp.py:249
2014-10-15 12:11:48.709 11918 DEBUG ironic.common.pxe_utils [-] Building PXE config for node 6adf9e06-8b0f-4aa9-81ca-d036a15ae45b create_pxe_config /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/pxe_utils.py:131
2014-10-15 12:11:48.716 11918 DEBUG ironic.drivers.modules.pxe [-] Fetching kernel and ramdisk for node 6adf9e06-8b0f-4aa9-81ca-d036a15ae45b _cache_tftp_images /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/pxe.py:336
2014-10-15 12:11:48.716 11918 DEBUG ironic.common.glance_service.base_image_service [-] Getting image metadata from glance. Image: 982dc3dd-270c-4b34-ad61-8c2416191fa8 _show /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/glance_service/base_image_service.py:185
2014-10-15 12:11:48.790 11918 DEBUG ironic.common.glance_service.base_image_service [-] Getting image metadata from glance. Image: d2d7e186-dee7-4147-83a8-fff4267c309a _show /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/glance_service/base_image_service.py:185
2014-10-15 12:11:48.841 11918 DEBUG ironic.common.glance_service.base_image_service [-] Getting image metadata from glance. Image: baff7d11-e69c-49ff-bd26-2c22e0585564 _show /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/glance_service/base_image_service.py:185
2014-10-15 12:11:48.892 11918 DEBUG ironic.common.glance_service.base_image_service [-] Getting image metadata from glance. Image: 17e83e20-e402-4f84-8980-d87efb8259c7 _show /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/glance_service/base_image_service.py:185
2014-10-15 12:11:48.944 11918 DEBUG ironic.drivers.modules.image_cache [-] Destination /tftpboot/6adf9e06-8b0f-4aa9-81ca-d036a15ae45b/ramdisk already exists for image 982dc3dd-270c-4b34-ad61-8c2416191fa8 fetch_image /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/image_cache.py:104
2014-10-15 12:11:48.944 11918 DEBUG ironic.drivers.modules.image_cache [-] Destination /tftpboot/6adf9e06-8b0f-4aa9-81ca-d036a15ae45b/kernel already exists for image d2d7e186-dee7-4147-83a8-fff4267c309a fetch_image /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/image_cache.py:104
2014-10-15 12:11:48.945 11918 DEBUG ironic.drivers.modules.image_cache [-] Destination /tftpboot/6adf9e06-8b0f-4aa9-81ca-d036a15ae45b/deploy_ramdisk already exists for image baff7d11-e69c-49ff-bd26-2c22e0585564 fetch_image /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/image_cache.py:104
2014-10-15 12:11:48.945 11918 DEBUG ironic.drivers.modules.image_cache [-] Destination /tftpboot/6adf9e06-8b0f-4aa9-81ca-d036a15ae45b/deploy_kernel already exists for image 17e83e20-e402-4f84-8980-d87efb8259c7 fetch_image /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/image_cache.py:104
2014-10-15 12:11:48.945 11918 DEBUG ironic.drivers.modules.pxe [-] Fetching image b588925c-e866-404e-9de9-319db44520a7 for node 6adf9e06-8b0f-4aa9-81ca-d036a15ae45b _cache_instance_image /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/pxe.py:363
2014-10-15 12:11:48.946 11918 DEBUG ironic.common.glance_service.base_image_service [-] Getting image metadata from glance. Image: b588925c-e866-404e-9de9-319db44520a7 _show /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/glance_service/base_image_service.py:185
2014-10-15 12:11:49.002 11918 INFO ironic.drivers.modules.image_cache [-] Master cache miss for image b588925c-e866-404e-9de9-319db44520a7, starting download
2014-10-15 12:12:01.556 11918 DEBUG ironic.common.utils [-] Execution completed, command line is "env LC_ALL=C LANG=C qemu-img info /mnt/state/var/lib/ironic/master_images/tmpV4xJgc/b588925c-e866-404e-9de9-319db44520a7.part" execute /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/utils.py:65
2014-10-15 12:12:01.556 11918 DEBUG ironic.common.utils [-] Command stdout is: "image: /mnt/state/var/lib/ironic/master_images/tmpV4xJgc/b588925c-e866-404e-9de9-319db44520a7.part
file format: qcow2
virtual size: 4.3G (4622974976 bytes)
disk size: 1.2G
cluster_size: 65536
Format specific information:
    compat: 1.1
    lazy refcounts: false
" execute /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/utils.py:66
2014-10-15 12:12:01.557 11918 DEBUG ironic.common.utils [-] Command stderr is: "" execute /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/utils.py:67
2014-10-15 12:12:01.557 11918 DEBUG ironic.common.images [-] b588925c-e866-404e-9de9-319db44520a7 was qcow2, converting to raw image_to_raw /opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/images.py:216
2014-10-15 12:12:13.292 11918 WARNING ironic.conductor.manager [-] Error in deploy of node 6adf9e06-8b0f-4aa9-81ca-d036a15ae45b: Unexpected error while running command.
Command: qemu-img convert -O raw /mnt/state/var/lib/ironic/master_images/tmpV4xJgc/b588925c-e866-404e-9de9-319db44520a7.part /mnt/state/var/lib/ironic/master_images/tmpV4xJgc/b588925c-e866-404e-9de9-319db44520a7.converted
Exit code: 1
Stdout: ''
Stderr: 'qemu-img: error while writing sector 2734592: No space left on device\n'
Traceback (most recent call last):
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 455, in fire_timers
    timer()
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/eventlet/hubs/timer.py", line 58, in __call__
    cb(*args, **kw)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/eventlet/greenthread.py", line 212, in main
    result = function(*args, **kwargs)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/conductor/manager.py", line 475, in _do_node_deploy
    node.target_provision_state = states.NOSTATE
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/openstack/common/excutils.py", line 82, in __exit__
    six.reraise(self.type_, self.value, self.tb)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/conductor/manager.py", line 468, in _do_node_deploy
    new_state = task.driver.deploy.deploy(task)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/conductor/task_manager.py", line 118, in wrapper
    return f(*args, **kwargs)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/pxe.py", line 519, in deploy
    _cache_instance_image(task.context, task.node)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/pxe.py", line 365, in _cache_instance_image
    _fetch_images(ctx, InstanceImageCache(), [(uuid, image_path)])
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/pxe.py", line 328, in _fetch_images
    cache.fetch_image(uuid, path, ctx=ctx)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/image_cache.py", line 120, in fetch_image
    self._download_image(uuid, master_path, dest_path, ctx=ctx)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/drivers/modules/image_cache.py", line 140, in _download_image
    self._image_service)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/images.py", line 193, in fetch_to_raw
    image_to_raw(image_href, path, path_tmp)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/images.py", line 218, in image_to_raw
    convert_image(path_tmp, staged, 'raw')
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/images.py", line 174, in convert_image
    utils.execute(*cmd, run_as_root=run_as_root)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/common/utils.py", line 63, in execute
    result = processutils.execute(*cmd, **kwargs)
  File "/opt/stack/venvs/openstack/local/lib/python2.7/site-packages/ironic/openstack/common/processutils.py", line 199, in execute
    cmd=' '.join(cmd))
ProcessExecutionError: Unexpected error while running command.
Command: qemu-img convert -O raw /mnt/state/var/lib/ironic/master_images/tmpV4xJgc/b588925c-e866-404e-9de9-319db44520a7.part /mnt/state/var/lib/ironic/master_images/tmpV4xJgc/b588925c-e866-404e-9de9-319db44520a7.converted
Exit code: 1
Stdout: ''
Stderr: 'qemu-img: error while writing sector 2734592: No space left on device\n'