I configured my setup to work with swift backend.
my data servers do not have space left.
when I create an image with --location, the image is created locally and glance is not trying to copy it to swift until we boot an instance.
since there is no space left on swift, when I try to boot, the instance is simply stuck in scheduling forever.
[root@opens-vdsb ~(keystone_admin)]# nova list
+--------------------------------------+------+--------+------------+-------------+----------+
| ID | Name | Status | Task State | Power State | Networks |
+--------------------------------------+------+--------+------------+-------------+----------+
| d6ae7ce8-0689-4580-b65d-29dae8497999 | bla | BUILD | scheduling | NOSTATE | |
+--------------------------------------+------+--------+------------+-------------+----------+
[root@opens-vdsb ~(keystone_admin)]#
[root@opens-vdsb ~(keystone_admin)]# egrep d6ae7ce8-0689-4580-b65d-29dae8497999 /var/log/*/*
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:35:31 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:35:34 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:35:36 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:35:41 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:35:49 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:35:59 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:36:11 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:36:26 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:36:44 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 684 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:04 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 684 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:06 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 684 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:11 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 684 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:19 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 684 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:29 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:41 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:44 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:49 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:37:56 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:38:06 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:38:19 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:38:34 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 687 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:38:51 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 687 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:38:54 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 687 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:38:59 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 687 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:39:06 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 687 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:39:16 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 687 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:39:34 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/httpd/access_log:10.35.200.226 - - [28/Oct/2013:14:39:37 +0200] "GET /dashboard/project/instances/?action=row_update&table=instances&obj_id=d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" 200 686 "http://opens-vdsb.qa.lab.tlv.redhat.com/dashboard/project/instances/" "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130806 Firefox/17.0"
/var/log/nova/api.log:2013-10-28 14:35:29.503 6058 DEBUG nova.compute.api [req-8e216578-df6c-49e7-838f-4ce9eba08f70 e77696a491084a0b86ff8c7f0bd718f2 3d491c71c61944f2a6b0b935ce6665f9] [instance: d6ae7ce8-0689-4580-b65d-29dae8497999] block_device_mapping [{'guest_format': None, 'boot_index': 0, 'no_device': None, 'connection_info': None, 'snapshot_id': None, 'volume_size': None, 'device_name': None, 'disk_bus': None, 'image_id': u'fecf77fd-aa3d-4a4d-b07c-7391bc085f90', 'source_type': 'image', 'device_type': 'disk', 'volume_id': None, 'destination_type': 'local', 'delete_on_termination': True}] _update_block_device_mapping /usr/lib/python2.6/site-packages/nova/compute/api.py:946
/var/log/nova/api.log:2013-10-28 14:35:29.742 6058 DEBUG qpid.messaging.io.ops [-] SENT[6f94cb0]: MessageTransfer(destination='nova', id=serial(2), sync=True, headers=(DeliveryProperties(routing_key='conductor'), MessageProperties(content_type='amqp/map', application_headers={'qpid.subject': 'conductor'})), payload='\x00\x00\x12}\x00\x00\x00\x02\x0coslo.message\x95\x12V{"_context_roles": ["admin"], "_context_request_id": "req-8e216578-df6c-49e7-838f-4ce9eba08f70", "_context_quota_class": null, "_context_service_catalog": [{"endpoints": [{"adminURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9", "region": "RegionOne", "id": "040e7821ac784f0e9c05f2f9eb930730", "internalURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9", "publicURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9"}], "endpoints_links": [], "type": "volume", "name": "cinder"}], "_context_tenant": "3d491c71c61944f2a6b0b935ce6665f9", "args": {"legacy_bdm": false, "requested_networks": null, "block_device_mapping": [{"instance_uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "guest_format": null, "boot_index": 0, "no_device": null, "connection_info": null, "snapshot_id": null, "volume_size": null, "device_name": null, "disk_bus": null, "image_id": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "source_type": "image", "device_type": "disk", "volume_id": null, "destination_type": "local", "delete_on_termination": true}], "image": {"status": "active", "name": "test1", "deleted": false, "container_format": "bare", "created_at": "2013-10-28T11:11:35.000000", "disk_format": "qcow2", "updated_at": "2013-10-28T11:29:49.000000", "properties": {}, "min_disk": 0, "min_ram": 0, "checksum": null, "owner": "3d491c71c61944f2a6b0b935ce6665f9", "is_public": false, "deleted_at": null, "id": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "size": 6442450945}, "filter_properties": {"instance_type": {"memory_mb": 2048, "root_gb": 20, "deleted_at": null, "name": "m1.small", "deleted": 0, "created_at": null, "ephemeral_gb": 0, "updated_at": null, "disabled": false, "vcpus": 1, "extra_specs": {}, "swap": 0, "rxtx_factor": 1.0, "is_public": true, "flavorid": "2", "vcpu_weight": null, "id": 5}, "scheduler_hints": {}}, "instances": [{"vm_state": "building", "availability_zone": null, "terminated_at": null, "ephemeral_gb": 0, "instance_type_id": 5, "user_data": null, "cleaned": false, "vm_mode": null, "deleted_at": null, "reservation_id": "r-05ggj319", "id": 3, "security_groups": {"objects": [{"deleted_at": null, "user_id": "e77696a491084a0b86ff8c7f0bd718f2", "description": "default", "deleted": false, "created_at": "2013-10-28T11:30:08.000000", "updated_at": null, "project_id": "3d491c71c61944f2a6b0b935ce6665f9", "id": 1, "name": "default"}]}, "disable_terminate": false, "root_device_name": null, "display_name": "bla", "uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "default_swap_device": null, "info_cache": {"instance_uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "network_info": []}, "hostname": "bla", "launched_on": null, "display_description": "bla", "key_data": null, "deleted": false, "scheduled_at": null, "power_state": 0, "default_ephemeral_device": null, "progress": 0, "project_id": "3d491c71c61944f2a6b0b935ce6665f9", "launched_at": null, "config_drive": "", "node": null, "ramdisk_id": "", "access_ip_v6": null, "access_ip_v4": null, "kernel_id": "", "key_name": null, "updated_at": null, "host": null, "user_id": "e77696a491084a0b86ff8c7f0bd718f2", "system_metadata": {"instance_type_memory_mb": 2048, "instance_type_swap": 0, "instance_type_vcpu_weight": null, "instance_type_root_gb": 20, "instance_type_name": "m1.small", "instance_type_id": 5, "instance_type_ephemeral_gb": 0, "instance_type_rxtx_factor": 1.0, "image_disk_format": "qcow2", "instance_type_flavorid": "2", "instance_type_vcpus": 1, "image_container_format": "bare", "image_min_ram": 0, "image_min_disk": 20, "image_base_image_ref": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90"}, "task_state": "scheduling", "shutdown_terminate": false, "cell_name": null, "root_gb": 20, "locked": false, "name": "instance-00000003", "created_at": "2013-10-28T12:35:29.324767", "locked_by": null, "launch_index": 0, "memory_mb": 2048, "vcpus": 1, "image_ref": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "architecture": null, "auto_disk_config": false, "os_type": null, "metadata": {}}], "admin_password": "4LfA5zcyFR5v", "injected_files": [], "security_groups": ["default"]}, "_unique_id": "abd8e2ba05224272a5843bc8e7821233", "_context_timestamp": "2013-10-28T12:35:29.096725", "_context_user_id": "e77696a491084a0b86ff8c7f0bd718f2", "_context_project_name": "admin", "_context_read_deleted": "no", "_context_auth_token": "67947b83b7c1857c5911ba52a9bcfa6d", "namespace": "compute_task", "_context_instance_lock_checked": false, "_context_is_admin": true, "version": "1.5", "_context_project_id": "3d491c71c61944f2a6b0b935ce6665f9", "_context_user": "e77696a491084a0b86ff8c7f0bd718f2", "_context_user_name": "admin", "method": "build_instances", "_context_remote_address": "10.35.101.10"}\x0coslo.version\x95\x00\x032.0') write_op /usr/lib/python2.6/site-packages/qpid/messaging/driver.py:686
/var/log/nova/api.log:2013-10-28 14:35:29.742 6058 DEBUG qpid.messaging [-] SENT[69d8128]: Message(properties={'qpid.subject': 'conductor'}, content={'oslo.message': '{"_context_roles": ["admin"], "_context_request_id": "req-8e216578-df6c-49e7-838f-4ce9eba08f70", "_context_quota_class": null, "_context_service_catalog": [{"endpoints": [{"adminURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9", "region": "RegionOne", "id": "040e7821ac784f0e9c05f2f9eb930730", "internalURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9", "publicURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9"}], "endpoints_links": [], "type": "volume", "name": "cinder"}], "_context_tenant": "3d491c71c61944f2a6b0b935ce6665f9", "args": {"legacy_bdm": false, "requested_networks": null, "block_device_mapping": [{"instance_uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "guest_format": null, "boot_index": 0, "no_device": null, "connection_info": null, "snapshot_id": null, "volume_size": null, "device_name": null, "disk_bus": null, "image_id": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "source_type": "image", "device_type": "disk", "volume_id": null, "destination_type": "local", "delete_on_termination": true}], "image": {"status": "active", "name": "test1", "deleted": false, "container_format": "bare", "created_at": "2013-10-28T11:11:35.000000", "disk_format": "qcow2", "updated_at": "2013-10-28T11:29:49.000000", "properties": {}, "min_disk": 0, "min_ram": 0, "checksum": null, "owner": "3d491c71c61944f2a6b0b935ce6665f9", "is_public": false, "deleted_at": null, "id": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "size": 6442450945}, "filter_properties": {"instance_type": {"memory_mb": 2048, "root_gb": 20, "deleted_at": null, "name": "m1.small", "deleted": 0, "created_at": null, "ephemeral_gb": 0, "updated_at": null, "disabled": false, "vcpus": 1, "extra_specs": {}, "swap": 0, "rxtx_factor": 1.0, "is_public": true, "flavorid": "2", "vcpu_weight": null, "id": 5}, "scheduler_hints": {}}, "instances": [{"vm_state": "building", "availability_zone": null, "terminated_at": null, "ephemeral_gb": 0, "instance_type_id": 5, "user_data": null, "cleaned": false, "vm_mode": null, "deleted_at": null, "reservation_id": "r-05ggj319", "id": 3, "security_groups": {"objects": [{"deleted_at": null, "user_id": "e77696a491084a0b86ff8c7f0bd718f2", "description": "default", "deleted": false, "created_at": "2013-10-28T11:30:08.000000", "updated_at": null, "project_id": "3d491c71c61944f2a6b0b935ce6665f9", "id": 1, "name": "default"}]}, "disable_terminate": false, "root_device_name": null, "display_name": "bla", "uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "default_swap_device": null, "info_cache": {"instance_uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "network_info": []}, "hostname": "bla", "launched_on": null, "display_description": "bla", "key_data": null, "deleted": false, "scheduled_at": null, "power_state": 0, "default_ephemeral_device": null, "progress": 0, "project_id": "3d491c71c61944f2a6b0b935ce6665f9", "launched_at": null, "config_drive": "", "node": null, "ramdisk_id": "", "access_ip_v6": null, "access_ip_v4": null, "kernel_id": "", "key_name": null, "updated_at": null, "host": null, "user_id": "e77696a491084a0b86ff8c7f0bd718f2", "system_metadata": {"instance_type_memory_mb": 2048, "instance_type_swap": 0, "instance_type_vcpu_weight": null, "instance_type_root_gb": 20, "instance_type_name": "m1.small", "instance_type_id": 5, "instance_type_ephemeral_gb": 0, "instance_type_rxtx_factor": 1.0, "image_disk_format": "qcow2", "instance_type_flavorid": "2", "instance_type_vcpus": 1, "image_container_format": "bare", "image_min_ram": 0, "image_min_disk": 20, "image_base_image_ref": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90"}, "task_state": "scheduling", "shutdown_terminate": false, "cell_name": null, "root_gb": 20, "locked": false, "name": "instance-00000003", "created_at": "2013-10-28T12:35:29.324767", "locked_by": null, "launch_index": 0, "memory_mb": 2048, "vcpus": 1, "image_ref": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "architecture": null, "auto_disk_config": false, "os_type": null, "metadata": {}}], "admin_password": "4LfA5zcyFR5v", "injected_files": [], "security_groups": ["default"]}, "_unique_id": "abd8e2ba05224272a5843bc8e7821233", "_context_timestamp": "2013-10-28T12:35:29.096725", "_context_user_id": "e77696a491084a0b86ff8c7f0bd718f2", "_context_project_name": "admin", "_context_read_deleted": "no", "_context_auth_token": "67947b83b7c1857c5911ba52a9bcfa6d", "namespace": "compute_task", "_context_instance_lock_checked": false, "_context_is_admin": true, "version": "1.5", "_context_project_id": "3d491c71c61944f2a6b0b935ce6665f9", "_context_user": "e77696a491084a0b86ff8c7f0bd718f2", "_context_user_name": "admin", "method": "build_instances", "_context_remote_address": "10.35.101.10"}', 'oslo.version': '2.0'}) send /usr/lib/python2.6/site-packages/qpid/messaging/driver.py:1280
/var/log/nova/api.log:2013-10-28 14:35:29.743 6058 DEBUG qpid.messaging.io.raw [-] SENT[6f94cb0]: '\x0b\x01\x00\x17\x00\x01\x00\x00\x00\x00\x00\x00\x04\x01\x01\x01\x01\x00\x04nova\x03\x02\x00P\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\x04\x01\x00\x02\tconductor\x00\x00\x00.\x04\x03\x10\x01\x08amqp/map\x00\x00\x00\x1d\x00\x00\x00\x01\x0cqpid.subject\x95\x00\tconductor\x07\x03\x12\x8d\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x12}\x00\x00\x00\x02\x0coslo.message\x95\x12V{"_context_roles": ["admin"], "_context_request_id": "req-8e216578-df6c-49e7-838f-4ce9eba08f70", "_context_quota_class": null, "_context_service_catalog": [{"endpoints": [{"adminURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9", "region": "RegionOne", "id": "040e7821ac784f0e9c05f2f9eb930730", "internalURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9", "publicURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9"}], "endpoints_links": [], "type": "volume", "name": "cinder"}], "_context_tenant": "3d491c71c61944f2a6b0b935ce6665f9", "args": {"legacy_bdm": false, "requested_networks": null, "block_device_mapping": [{"instance_uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "guest_format": null, "boot_index": 0, "no_device": null, "connection_info": null, "snapshot_id": null, "volume_size": null, "device_name": null, "disk_bus": null, "image_id": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "source_type": "image", "device_type": "disk", "volume_id": null, "destination_type": "local", "delete_on_termination": true}], "image": {"status": "active", "name": "test1", "deleted": false, "container_format": "bare", "created_at": "2013-10-28T11:11:35.000000", "disk_format": "qcow2", "updated_at": "2013-10-28T11:29:49.000000", "properties": {}, "min_disk": 0, "min_ram": 0, "checksum": null, "owner": "3d491c71c61944f2a6b0b935ce6665f9", "is_public": false, "deleted_at": null, "id": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "size": 6442450945}, "filter_properties": {"instance_type": {"memory_mb": 2048, "root_gb": 20, "deleted_at": null, "name": "m1.small", "deleted": 0, "created_at": null, "ephemeral_gb": 0, "updated_at": null, "disabled": false, "vcpus": 1, "extra_specs": {}, "swap": 0, "rxtx_factor": 1.0, "is_public": true, "flavorid": "2", "vcpu_weight": null, "id": 5}, "scheduler_hints": {}}, "instances": [{"vm_state": "building", "availability_zone": null, "terminated_at": null, "ephemeral_gb": 0, "instance_type_id": 5, "user_data": null, "cleaned": false, "vm_mode": null, "deleted_at": null, "reservation_id": "r-05ggj319", "id": 3, "security_groups": {"objects": [{"deleted_at": null, "user_id": "e77696a491084a0b86ff8c7f0bd718f2", "description": "default", "deleted": false, "created_at": "2013-10-28T11:30:08.000000", "updated_at": null, "project_id": "3d491c71c61944f2a6b0b935ce6665f9", "id": 1, "name": "default"}]}, "disable_terminate": false, "root_device_name": null, "display_name": "bla", "uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "default_swap_device": null, "info_cache": {"instance_uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "network_info": []}, "hostname": "bla", "launched_on": null, "display_description": "bla", "key_data": null, "deleted": false, "scheduled_at": null, "power_state": 0, "default_ephemeral_device": null, "progress": 0, "project_id": "3d491c71c61944f2a6b0b935ce6665f9", "launched_at": null, "config_drive": "", "node": null, "ramdisk_id": "", "access_ip_v6": null, "access_ip_v4": null, "kernel_id": "", "key_name": null, "updated_at": null, "host": null, "user_id": "e77696a491084a0b86ff8c7f0bd718f2", "system_metadata": {"instance_type_memory_mb": 2048, "instance_type_swap": 0, "instance_type_vcpu_weight": null, "instance_type_root_gb": 20, "instance_type_name": "m1.small", "instance_type_id": 5, "instance_type_ephemeral_gb": 0, "instance_type_rxtx_factor": 1.0, "image_disk_format": "qcow2", "instance_type_flavorid": "2", "instance_type_vcpus": 1, "image_container_format": "bare", "image_min_ram": 0, "image_min_disk": 20, "image_base_image_ref": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90"}, "task_state": "scheduling", "shutdown_terminate": false, "cell_name": null, "root_gb": 20, "locked": false, "name": "instance-00000003", "created_at": "2013-10-28T12:35:29.324767", "locked_by": null, "launch_index": 0, "memory_mb": 2048, "vcpus": 1, "image_ref": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "architecture": null, "auto_disk_config": false, "os_type": null, "metadata": {}}], "admin_password": "4LfA5zcyFR5v", "injected_files": [], "security_groups": ["default"]}, "_unique_id": "abd8e2ba05224272a5843bc8e7821233", "_context_timestamp": "2013-10-28T12:35:29.096725", "_context_user_id": "e77696a491084a0b86ff8c7f0bd718f2", "_context_project_name": "admin", "_context_read_deleted": "no", "_context_auth_token": "67947b83b7c1857c5911ba52a9bcfa6d", "namespace": "compute_task", "_context_instance_lock_checked": false, "_context_is_admin": true, "version": "1.5", "_context_project_id": "3d491c71c61944f2a6b0b935ce6665f9", "_context_user": "e77696a491084a0b86ff8c7f0bd718f2", "_context_user_name": "admin", "method": "build_instances", "_context_remote_address": "10.35.101.10"}\x0coslo.version\x95\x00\x032.0' writeable /usr/lib/python2.6/site-packages/qpid/messaging/driver.py:480
/var/log/nova/api.log:2013-10-28 14:35:29.744 6058 DEBUG qpid.messaging [-] RACK[69d8128]: Message(properties={'qpid.subject': 'conductor'}, content={'oslo.message': '{"_context_roles": ["admin"], "_context_request_id": "req-8e216578-df6c-49e7-838f-4ce9eba08f70", "_context_quota_class": null, "_context_service_catalog": [{"endpoints": [{"adminURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9", "region": "RegionOne", "id": "040e7821ac784f0e9c05f2f9eb930730", "internalURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9", "publicURL": "http://10.35.101.10:8776/v1/3d491c71c61944f2a6b0b935ce6665f9"}], "endpoints_links": [], "type": "volume", "name": "cinder"}], "_context_tenant": "3d491c71c61944f2a6b0b935ce6665f9", "args": {"legacy_bdm": false, "requested_networks": null, "block_device_mapping": [{"instance_uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "guest_format": null, "boot_index": 0, "no_device": null, "connection_info": null, "snapshot_id": null, "volume_size": null, "device_name": null, "disk_bus": null, "image_id": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "source_type": "image", "device_type": "disk", "volume_id": null, "destination_type": "local", "delete_on_termination": true}], "image": {"status": "active", "name": "test1", "deleted": false, "container_format": "bare", "created_at": "2013-10-28T11:11:35.000000", "disk_format": "qcow2", "updated_at": "2013-10-28T11:29:49.000000", "properties": {}, "min_disk": 0, "min_ram": 0, "checksum": null, "owner": "3d491c71c61944f2a6b0b935ce6665f9", "is_public": false, "deleted_at": null, "id": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "size": 6442450945}, "filter_properties": {"instance_type": {"memory_mb": 2048, "root_gb": 20, "deleted_at": null, "name": "m1.small", "deleted": 0, "created_at": null, "ephemeral_gb": 0, "updated_at": null, "disabled": false, "vcpus": 1, "extra_specs": {}, "swap": 0, "rxtx_factor": 1.0, "is_public": true, "flavorid": "2", "vcpu_weight": null, "id": 5}, "scheduler_hints": {}}, "instances": [{"vm_state": "building", "availability_zone": null, "terminated_at": null, "ephemeral_gb": 0, "instance_type_id": 5, "user_data": null, "cleaned": false, "vm_mode": null, "deleted_at": null, "reservation_id": "r-05ggj319", "id": 3, "security_groups": {"objects": [{"deleted_at": null, "user_id": "e77696a491084a0b86ff8c7f0bd718f2", "description": "default", "deleted": false, "created_at": "2013-10-28T11:30:08.000000", "updated_at": null, "project_id": "3d491c71c61944f2a6b0b935ce6665f9", "id": 1, "name": "default"}]}, "disable_terminate": false, "root_device_name": null, "display_name": "bla", "uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "default_swap_device": null, "info_cache": {"instance_uuid": "d6ae7ce8-0689-4580-b65d-29dae8497999", "network_info": []}, "hostname": "bla", "launched_on": null, "display_description": "bla", "key_data": null, "deleted": false, "scheduled_at": null, "power_state": 0, "default_ephemeral_device": null, "progress": 0, "project_id": "3d491c71c61944f2a6b0b935ce6665f9", "launched_at": null, "config_drive": "", "node": null, "ramdisk_id": "", "access_ip_v6": null, "access_ip_v4": null, "kernel_id": "", "key_name": null, "updated_at": null, "host": null, "user_id": "e77696a491084a0b86ff8c7f0bd718f2", "system_metadata": {"instance_type_memory_mb": 2048, "instance_type_swap": 0, "instance_type_vcpu_weight": null, "instance_type_root_gb": 20, "instance_type_name": "m1.small", "instance_type_id": 5, "instance_type_ephemeral_gb": 0, "instance_type_rxtx_factor": 1.0, "image_disk_format": "qcow2", "instance_type_flavorid": "2", "instance_type_vcpus": 1, "image_container_format": "bare", "image_min_ram": 0, "image_min_disk": 20, "image_base_image_ref": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90"}, "task_state": "scheduling", "shutdown_terminate": false, "cell_name": null, "root_gb": 20, "locked": false, "name": "instance-00000003", "created_at": "2013-10-28T12:35:29.324767", "locked_by": null, "launch_index": 0, "memory_mb": 2048, "vcpus": 1, "image_ref": "fecf77fd-aa3d-4a4d-b07c-7391bc085f90", "architecture": null, "auto_disk_config": false, "os_type": null, "metadata": {}}], "admin_password": "4LfA5zcyFR5v", "injected_files": [], "security_groups": ["default"]}, "_unique_id": "abd8e2ba05224272a5843bc8e7821233", "_context_timestamp": "2013-10-28T12:35:29.096725", "_context_user_id": "e77696a491084a0b86ff8c7f0bd718f2", "_context_project_name": "admin", "_context_read_deleted": "no", "_context_auth_token": "67947b83b7c1857c5911ba52a9bcfa6d", "namespace": "compute_task", "_context_instance_lock_checked": false, "_context_is_admin": true, "version": "1.5", "_context_project_id": "3d491c71c61944f2a6b0b935ce6665f9", "_context_user": "e77696a491084a0b86ff8c7f0bd718f2", "_context_user_name": "admin", "method": "build_instances", "_context_remote_address": "10.35.101.10"}', 'oslo.version': '2.0'}) msg_acked /usr/lib/python2.6/site-packages/qpid/messaging/driver.py:1269
/var/log/nova/api.log:2013-10-28 14:35:31.593 6058 DEBUG routes.middleware [-] Matched GET /3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:100
/var/log/nova/api.log:2013-10-28 14:35:31.593 6058 DEBUG routes.middleware [-] Match dict: {'action': u'show', 'controller': <nova.api.openstack.wsgi.Resource object at 0x46b9290>, 'project_id': u'3d491c71c61944f2a6b0b935ce6665f9', 'id': u'd6ae7ce8-0689-4580-b65d-29dae8497999'} __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:103
/var/log/nova/api.log:2013-10-28 14:35:31.645 6058 INFO nova.osapi_compute.wsgi.server [req-2275fd95-92ba-47a3-9f59-c2a1f54069b3 e77696a491084a0b86ff8c7f0bd718f2 3d491c71c61944f2a6b0b935ce6665f9] 10.35.101.10 "GET /v2/3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" status: 200 len: 1638 time: 0.0571580
/var/log/nova/api.log:2013-10-28 14:35:34.063 6058 DEBUG routes.middleware [-] Matched GET /3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:100
/var/log/nova/api.log:2013-10-28 14:35:34.064 6058 DEBUG routes.middleware [-] Match dict: {'action': u'show', 'controller': <nova.api.openstack.wsgi.Resource object at 0x46b9290>, 'project_id': u'3d491c71c61944f2a6b0b935ce6665f9', 'id': u'd6ae7ce8-0689-4580-b65d-29dae8497999'} __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:103
/var/log/nova/api.log:2013-10-28 14:35:34.117 6058 INFO nova.osapi_compute.wsgi.server [req-58eb538c-7429-4483-afa8-585a2f6979e2 e77696a491084a0b86ff8c7f0bd718f2 3d491c71c61944f2a6b0b935ce6665f9] 10.35.101.10 "GET /v2/3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" status: 200 len: 1638 time: 0.0585499
/var/log/nova/api.log:2013-10-28 14:35:36.686 6058 DEBUG routes.middleware [-] Matched GET /3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:100
/var/log/nova/api.log:2013-10-28 14:35:36.687 6058 DEBUG routes.middleware [-] Match dict: {'action': u'show', 'controller': <nova.api.openstack.wsgi.Resource object at 0x46b9290>, 'project_id': u'3d491c71c61944f2a6b0b935ce6665f9', 'id': u'd6ae7ce8-0689-4580-b65d-29dae8497999'} __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:103
/var/log/nova/api.log:2013-10-28 14:35:36.738 6058 INFO nova.osapi_compute.wsgi.server [req-dbfa62f6-8311-43ea-b851-5282c51ea00c e77696a491084a0b86ff8c7f0bd718f2 3d491c71c61944f2a6b0b935ce6665f9] 10.35.101.10 "GET /v2/3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" status: 200 len: 1638 time: 0.0577772
/var/log/nova/api.log:2013-10-28 14:35:41.708 6058 DEBUG routes.middleware [-] Matched GET /3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:100
/var/log/nova/api.log:2013-10-28 14:35:41.708 6058 DEBUG routes.middleware [-] Match dict: {'action': u'show', 'controller': <nova.api.openstack.wsgi.Resource object at 0x46b9290>, 'project_id': u'3d491c71c61944f2a6b0b935ce6665f9', 'id': u'd6ae7ce8-0689-4580-b65d-29dae8497999'} __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:103
/var/log/nova/api.log:2013-10-28 14:35:41.762 6058 INFO nova.osapi_compute.wsgi.server [req-71b1b2d9-8668-4a8e-9cf2-6555168b123a e77696a491084a0b86ff8c7f0bd718f2 3d491c71c61944f2a6b0b935ce6665f9] 10.35.101.10 "GET /v2/3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" status: 200 len: 1638 time: 0.0594401
/var/log/nova/api.log:2013-10-28 14:35:49.221 6058 DEBUG routes.middleware [-] Matched GET /3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:100
/var/log/nova/api.log:2013-10-28 14:35:49.221 6058 DEBUG routes.middleware [-] Match dict: {'action': u'show', 'controller': <nova.api.openstack.wsgi.Resource object at 0x46b9290>, 'project_id': u'3d491c71c61944f2a6b0b935ce6665f9', 'id': u'd6ae7ce8-0689-4580-b65d-29dae8497999'} __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:103
/var/log/nova/api.log:2013-10-28 14:35:49.276 6058 INFO nova.osapi_compute.wsgi.server [req-7ff476e8-4233-4712-aeb4-2e9e60c1cf5b e77696a491084a0b86ff8c7f0bd718f2 3d491c71c61944f2a6b0b935ce6665f9] 10.35.101.10 "GET /v2/3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" status: 200 len: 1638 time: 0.0620940
/var/log/nova/api.log:2013-10-28 14:35:59.243 6058 DEBUG routes.middleware [-] Matched GET /3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:100
/var/log/nova/api.log:2013-10-28 14:35:59.244 6058 DEBUG routes.middleware [-] Match dict: {'action': u'show', 'controller': <nova.api.openstack.wsgi.Resource object at 0x46b9290>, 'project_id': u'3d491c71c61944f2a6b0b935ce6665f9', 'id': u'd6ae7ce8-0689-4580-b65d-29dae8497999'} __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:103
/var/log/nova/api.log:2013-10-28 14:35:59.295 6058 INFO nova.osapi_compute.wsgi.server [req-8e7b5701-340c-4382-96f4-086ccd804288 e77696a491084a0b86ff8c7f0bd718f2 3d491c71c61944f2a6b0b935ce6665f9] 10.35.101.10 "GET /v2/3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 HTTP/1.1" status: 200 len: 1638 time: 0.0570891
/var/log/nova/api.log:2013-10-28 14:36:11.701 6058 DEBUG routes.middleware [-] Matched GET /3d491c71c61944f2a6b0b935ce6665f9/servers/d6ae7ce8-0689-4580-b65d-29dae8497999 __call__ /usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py:100
The big data dump isn't really helpful. Are there errors in the nova compute or scheduler logs? Those would probably be most helpful to know where this fails and how to make sure it gets cleaned up from the scheduler.