After upgrading a source installation from stable/juno to stable/kilo, I get the following error when trying to terminate a volume backed instance:
Traceback (most recent call last):
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 134, in _dispatch_and_reply
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 177, in _dispatch
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 123, in _do_dispatch
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/osprofiler/profiler.py", line 105, in wrapper
return f(*args, **kwargs)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/cinder/volume/manager.py", line 969, in terminate_connection
self.db.volume_update(context, volume_id,
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/cinder/db/api.py", line 210, in volume_get
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/cinder/db/sqlalchemy/api.py", line 172, in wrapper
:py:func:`authorize_user_context`.
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/cinder/db/sqlalchemy/api.py", line 1227, in volume_get
volume_ref['status'] = 'in-use'
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/cinder/db/sqlalchemy/api.py", line 172, in wrapper
:py:func:`authorize_user_context`.
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/cinder/db/sqlalchemy/api.py", line 1216, in _volume_get
if not remain_attachment:
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2370, in first
ret = list(self[0:1])
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2231, in __getitem__
return list(res)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2441, in __iter__
return self._execute_and_instances(context)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2456, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 841, in execute
return meth(self, multiparams, params)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 938, in _execute_clauseelement
compiled_sql, distilled_params
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1070, in _execute_context
context)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1267, in _handle_dbapi_exception
util.raise_from_cause(newraise, exc_info)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1063, in _execute_context
context)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 442, in do_execute
cursor.execute(statement, parameters)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/cursors.py", line 134, in execute
result = self._query(query)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/cursors.py", line 282, in _query
conn.query(q)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/connections.py", line 768, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/connections.py", line 929, in _read_query_result
result.read()
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/connections.py", line 1125, in read
first_packet = self.connection._read_packet()
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/connections.py", line 893, in _read_packet
packet.check_error()
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/connections.py", line 369, in check_error
err.raise_mysql_exception(self._data)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/err.py", line 120, in raise_mysql_exception
_check_mysql_exception(errinfo)
File "/opt/openstack/venv/cinder/local/lib/python2.7/site-packages/pymysql/err.py", line 115, in _check_mysql_exception
raise InternalError(errno, errorvalue)
DBError: (InternalError) (1054, u"Unknown column 'volumes.instance_uuid' in 'field list'") 'SELECT anon_1.volumes_created_at AS anon_1_volumes_created_at, anon_1.volumes_updated_at AS anon_1_volumes_updated_at, anon_1.volumes_deleted_at AS anon_1_volumes_deleted_at, anon_1.volumes_id AS anon_1_volumes_id, anon_1.volumes__name_id AS anon_1_volumes__name_id, anon_1.volumes_ec2_id AS anon_1_volumes_ec2_id, anon_1.volumes_user_id AS anon_1_volumes_user_id, anon_1.volumes_project_id AS anon_1_volumes_project_id, anon_1.volumes_snapshot_id AS anon_1_volumes_snapshot_id, anon_1.volumes_host AS anon_1_volumes_host, anon_1.volumes_size AS anon_1_volumes_size, anon_1.volumes_availability_zone AS anon_1_volumes_availability_zone, anon_1.volumes_instance_uuid AS anon_1_volumes_instance_uuid, anon_1.volumes_attached_host AS anon_1_volumes_attached_host, anon_1.volumes_mountpoint AS anon_1_volumes_mountpoint, anon_1.volumes_attach_time AS anon_1_volumes_attach_time, anon_1.volumes_status AS anon_1_volumes_status, anon_1.volumes_attach_status AS anon_1_volumes_attach_status, anon_1.volumes_migration_status AS anon_1_volumes_migration_status, anon_1.volumes_scheduled_at AS anon_1_volumes_scheduled_at, anon_1.volumes_launched_at AS anon_1_volumes_launched_at, anon_1.volumes_terminated_at AS anon_1_volumes_terminated_at, anon_1.volumes_display_name AS anon_1_volumes_display_name, anon_1.volumes_display_description AS anon_1_volumes_display_description, anon_1.volumes_provider_location AS anon_1_volumes_provider_location, anon_1.volumes_provider_auth AS anon_1_volumes_provider_auth, anon_1.volumes_provider_geometry AS anon_1_volumes_provider_geometry, anon_1.volumes_volume_type_id AS anon_1_volumes_volume_type_id, anon_1.volumes_source_volid AS anon_1_volumes_source_volid, anon_1.volumes_encryption_key_id AS anon_1_volumes_encryption_key_id, anon_1.volumes_consistencygroup_id AS anon_1_volumes_consistencygroup_id, anon_1.volumes_deleted AS anon_1_volumes_deleted, anon_1.volumes_bootable AS anon_1_volumes_bootable, anon_1.volumes_replication_status AS anon_1_volumes_replication_status, anon_1.volumes_replication_extended_status AS anon_1_volumes_replication_extended_status, anon_1.volumes_replication_driver_data AS anon_1_volumes_replication_driver_data, consistencygroups_1.created_at AS consistencygroups_1_created_at, consistencygroups_1.updated_at AS consistencygroups_1_updated_at, consistencygroups_1.deleted_at AS consistencygroups_1_deleted_at, consistencygroups_1.deleted AS consistencygroups_1_deleted, consistencygroups_1.id AS consistencygroups_1_id, consistencygroups_1.user_id AS consistencygroups_1_user_id, consistencygroups_1.project_id AS consistencygroups_1_project_id, consistencygroups_1.host AS consistencygroups_1_host, consistencygroups_1.availability_zone AS consistencygroups_1_availability_zone, consistencygroups_1.name AS consistencygroups_1_name, consistencygroups_1.description AS consistencygroups_1_description, consistencygroups_1.volume_type_id AS consistencygroups_1_volume_type_id, consistencygroups_1.status AS consistencygroups_1_status, volume_types_1.created_at AS volume_types_1_created_at, volume_types_1.updated_at AS volume_types_1_updated_at, volume_types_1.deleted_at AS volume_types_1_deleted_at, volume_types_1.deleted AS volume_types_1_deleted, volume_types_1.id AS volume_types_1_id, volume_types_1.name AS volume_types_1_name, volume_types_1.qos_specs_id AS volume_types_1_qos_specs_id, volume_admin_metadata_1.created_at AS volume_admin_metadata_1_created_at, volume_admin_metadata_1.updated_at AS volume_admin_metadata_1_updated_at, volume_admin_metadata_1.deleted_at AS volume_admin_metadata_1_deleted_at, volume_admin_metadata_1.deleted AS volume_admin_metadata_1_deleted, volume_admin_metadata_1.id AS volume_admin_metadata_1_id, volume_admin_metadata_1.`key` AS volume_admin_metadata_1_key, volume_admin_metadata_1.value AS volume_admin_metadata_1_value, volume_admin_metadata_1.volume_id AS volume_admin_metadata_1_volume_id, volume_metadata_1.created_at AS volume_metadata_1_created_at, volume_metadata_1.updated_at AS volume_metadata_1_updated_at, volume_metadata_1.deleted_at AS volume_metadata_1_deleted_at, volume_metadata_1.deleted AS volume_metadata_1_deleted, volume_metadata_1.id AS volume_metadata_1_id, volume_metadata_1.`key` AS volume_metadata_1_key, volume_metadata_1.value AS volume_metadata_1_value, volume_metadata_1.volume_id AS volume_metadata_1_volume_id \nFROM (SELECT volumes.created_at AS volumes_created_at, volumes.updated_at AS volumes_updated_at, volumes.deleted_at AS volumes_deleted_at, volumes.id AS volumes_id, volumes._name_id AS volumes__name_id, volumes.ec2_id AS volumes_ec2_id, volumes.user_id AS volumes_user_id, volumes.project_id AS volumes_project_id, volumes.snapshot_id AS volumes_snapshot_id, volumes.host AS volumes_host, volumes.size AS volumes_size, volumes.availability_zone AS volumes_availability_zone, volumes.instance_uuid AS volumes_instance_uuid, volumes.attached_host AS volumes_attached_host, volumes.mountpoint AS volumes_mountpoint, volumes.attach_time AS volumes_attach_time, volumes.status AS volumes_status, volumes.attach_status AS volumes_attach_status, volumes.migration_status AS volumes_migration_status, volumes.scheduled_at AS volumes_scheduled_at, volumes.launched_at AS volumes_launched_at, volumes.terminated_at AS volumes_terminated_at, volumes.display_name AS volumes_display_name, volumes.display_description AS volumes_display_description, volumes.provider_location AS volumes_provider_location, volumes.provider_auth AS volumes_provider_auth, volumes.provider_geometry AS volumes_provider_geometry, volumes.volume_type_id AS volumes_volume_type_id, volumes.source_volid AS volumes_source_volid, volumes.encryption_key_id AS volumes_encryption_key_id, volumes.consistencygroup_id AS volumes_consistencygroup_id, volumes.deleted AS volumes_deleted, volumes.bootable AS volumes_bootable, volumes.replication_status AS volumes_replication_status, volumes.replication_extended_status AS volumes_replication_extended_status, volumes.replication_driver_data AS volumes_replication_driver_data \nFROM volumes \nWHERE volumes.deleted = 0 AND volumes.id = %s \n LIMIT %s) AS anon_1 LEFT OUTER JOIN consistencygroups AS consistencygroups_1 ON anon_1.volumes_consistencygroup_id = consistencygroups_1.id LEFT OUTER JOIN volume_types AS volume_types_1 ON anon_1.volumes_volume_type_id = volume_types_1.id AND volume_types_1.deleted = 0 LEFT OUTER JOIN volume_admin_metadata AS volume_admin_metadata_1 ON volume_admin_metadata_1.volume_id = anon_1.volumes_id AND volume_admin_metadata_1.deleted = 0 LEFT OUTER JOIN volume_metadata AS volume_metadata_1 ON volume_metadata_1.volume_id = anon_1.volumes_id AND volume_metadata_1.deleted = 0' ('8fcd0f2f-b7b0-4f1c-b4ec-416a455f48e1', 1)
$ cinder-manage db version
41
# Nothing new in HEAD https://github.com/openstack/cinder/tree/master/cinder/db/sqlalchemy/migrate_repo/versions
$ git branch
stable/juno
* stable/kilo
$ git rev-parse stable/kilo
ae6c589c52557988e24b5ee8005d8f81564a6068
Getting the same thing running periodic tasks too. Not sure if I've missed an upgrade step or what.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.
2015-04-20 11:33:10.029 40327 TRACE oslo.db.