All nova tables gone during lunch new compute node

[Yee]

Hi All
I am having below serious issues when setting up Multi node Devstack environment with Quantum + OpenVswitch.
Controller Node works well with below settings. all services including scheduler, quantum works great.
Controller Node localrc
=========
# -*- mode: shell-script -*-

# Setting OFFLINE to True switches off download of packages.
OFFLINE=False

# Setting RECLONE=yes will make stack.sh freshen each repo on each run.
RECLONE=yes

# Setting USE_TUNNELS=True will make OVS plugin run in tunnel mode.
USE_TUNNELS=True

# Destination path for installation of the OpenStack components.
# There is no need to specify it unless you want the code in
# some particular location (like in a directory shared by all VMs).
#DEST=/opt/stack

# Running DevStack with multiple hosts requires a custom localrc file
# for each host. The master is the same as a single host in installation
# with MULTI_HOST=True.
MULTI_HOST=True

# We do not run Nova CPU service, nor the Quantum L2/L3/DHCP agents on the
# controller node. However, Nova CPU service is enabled here, otherwise
# the stack.sh script does not setup things properly. Instead, we shutdown
# the Nova CPU service in local.sh.
disable_service n-net
#disable_service n-cpu
enable_service quantum
enable_service q-svc
#enable_service q-agt
#enable_service q-dhcp
#enable_service q-l3
# We only activate q-meta on the Controller to make the stack.sh
# enable the 'service_quantum_metadata_proxy' option in nova.conf.
# The quantum metadata agent will run on the Controller but will
# not be used.
enable_service q-meta

# For multinode setups we want a scheduler to distribute VMs.
SCHEDULER=nova.scheduler.simple.SimpleScheduler

# Use the OpenVSwitch plugin.
Q_PLUGIN=openvswitch

# My Controller/Network node has NATed access to the external world.
# Its default GW is 172.16.6.2 (provided by VMware Fusion), the VM host
# (my MBP) has IP address 172.16.6.1 (on vmnet8).
# eth0 - interface towards Internet
# eth1 - interface to management network
# eth2 - interface for tenant VM traffic

# The management network where my Devstack machines reside.
# ALWAYS VERIFY IT BEFORE RUNNING stack.sh!
MY_NET='172.16.1.0/24'

#######################################################################
# OpenVSwitch plugin settings
# ---------------------------

if [ "$USE_TUNNELS" == "True" ]; then
    # Set OVS plugin to (GRE) tunnel mode.
    ENABLE_TENANT_TUNNELS=True
    OVS_ENABLE_TUNNELING=True
    TENANT_TUNNEL_RANGE=500:1500
    ENABLE_TENANT_VLANS=False

    # The interface and bridge for tenant VM traffic.
    #GUEST_INTERFACE_DEFAULT=eth2

    tunnel_flag="-g"
else
    # Set OVS plugin to VLAN mode.
    ENABLE_TENANT_TUNNELS=False
    OVS_ENABLE_TUNNELING=False
    ENABLE_TENANT_VLANS=True
    TENANT_VLAN_RANGE=500:1500
    PHYSICAL_NETWORK=default

    # The interface for tenant VM traffic.
    #GUEST_INTERFACE_DEFAULT=eth2
    OVS_PHYSICAL_BRIDGE=br-eth2
    OVS_BRIDGE_MAPPINGS=$PHYSICAL_NETWORK:$OVS_PHYSICAL_BRIDGE

    tunnel_flag=""
fi

# Cleanup old stuff that may remain. Only useful if L3/DHCP agents
# run on the controller. Also, we only do this when stack.sh is
# the executed script.
#scriptname=`basename $0`
#if [ $scriptname == "stack.sh" ]; then
# sudo ~/bin/quantum_cleanup.sh
#fi

# If you plan to run Nova or L3/DHCP agents on the controller
# you need to perform the following to setup the interfaces and bridges
# that the 'stack.sh' script won't. Again we only do this if stack.sh
# is the executed script.
#if [ $scriptname == "stack.sh" ]; then
# sudo ~/bin/quantum_if_and_br_setup.sh -e eth0 $tunnel_flag -i eth2 -n 172.16.2.9/24
#fi

# End of OpenVSwitch plugin settings.
########################################################################

# ``HOST_IP`` is not set by default.
# This is the management network IP address of the Controller.
HOST_IP=`~/bin/hostitfc -i -c $MY_NET`

# The Internet facing IP address. Set it to IP number rather than name
# since we don't have names in DNS.
SERVICE_HOST=`awk '/ControllerPublic/ { print $1; } ' /etc/hosts`

# Settings to get NoVNC to work.
VNCSERVER_LISTEN=$HOST_IP
VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP

# Some network address settings relevant for my configuration.
FLAT_INTERFACE=`~/bin/hostitfc -n -c $MY_NET`
FIXED_RANGE=10.0.3.0/24
NETWORK_GATEWAY=10.0.3.1
FIXED_NETWORK_SIZE=256
FLOATING_RANGE=172.16.6.32/27

# Logging
# -------

# By default ``stack.sh`` output only goes to the terminal where it runs. It can
# be configured to additionally log to a file by setting ``LOGFILE`` to the full
# path of the destination log file. A timestamp will be appended to the given name.
LOGFILE=~/devstack/stack.sh.log

MYSQL_PASSWORD=simple
MYSQL_USER=stacker

RABBIT_PASSWORD=simple
SERVICE_TOKEN=simple
SERVICE_PASSWORD=simple
ADMIN_PASSWORD=simple
SERVICE_TOKEN=simple

# Type of virtualization to use. Options: kvm, lxc, qemu
LIBVIRT_TYPE=kvm
# Uncomment this to use LXC virtualization instead of (slower) QEMU.
#LIBVIRT_TYPE=lxc

# List of images to use.
case "$LIBVIRT_TYPE" in
    lxc) # the cirros root disk in the uec tarball is empty, so it will not work for lxc
 IMAGE_URLS="http://cloud-images.ubuntu.com/releases/oneiric/release/ubuntu-11.10-server-cloudimg-amd64.tar.gz,http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-rootfs.img.gz";;
    *) # otherwise, use the uec style image (with kernel, ramdisk, disk)
 IMAGE_URLS="http://cloud-images.ubuntu.com/releases/oneiric/release/ubuntu-11.10-server-cloudimg-amd64.tar.gz,http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-uec.tar.gz";;
esac

# Uncomment this to set the default instance type.
#DEFAULT_INSTANCE_TYPE=m1.micro
=======================================================
Controller Node nova.conf
===========================================
[DEFAULT]
verbose=True
auth_strategy=keystone
allow_resize_to_same_host=True
api_paste_config=/etc/nova/api-paste.ini
rootwrap_config=/etc/nova/rootwrap.conf
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
dhcpbridge_flagfile=/etc/nova/nova.conf
force_dhcp_release=True
fixed_range=10.0.3.0/24
s3_host=192.168.255.141
s3_port=3333
osapi_compute_extension=nova.api.openstack.compute.contrib.standard_extensions
my_ip=192.168.255.141
sql_connection=mysql://stacker:simple@localhost/nova?charset=utf8
libvirt_type=kvm
libvirt_cpu_mode=none
instance_name_template=instance-%08x
enabled_apis=ec2,osapi_compute,metadata
volume_api_class=nova.volume.cinder.API
state_path=/opt/stack/data/nova
lock_path=/opt/stack/data/nova
instances_path=/opt/stack/data/nova/instances
multi_host=True
send_arp_for_ha=True
use_syslog=True
logging_context_format_string=%(asctime)s %(levelname)s %(name)s [%(request_id)s
%(user_name)s %(project_name)s] %(instance)s%(message)s
network_api_class=nova.network.quantumv2.api.API
quantum_admin_username=quantum
quantum_admin_password=simple
quantum_admin_auth_url=http://192.168.255.141:35357/v2.0
quantum_auth_strategy=keystone
quantum_admin_tenant_name=service
quantum_url=http://192.168.255.141:9696
libvirt_vif_driver=nova.virt.libvirt.vif.LibvirtHybridOVSBridgeDriver
linuxnet_interface_driver=
service_quantum_metadata_proxy=True
novncproxy_base_url=http://192.168.255.141:6080/vnc_auto.html
xvpvncproxy_base_url=http://192.168.255.141:6081/console
vncserver_listen=
vncserver_proxyclient_address=
ec2_dmz_host=192.168.255.141
rabbit_host=localhost
rabbit_password=simple
glance_api_servers=192.168.255.141:9292
compute_driver=libvirt.LibvirtDriver
firewall_driver=nova.virt.libvirt.firewall.IptablesFirewallDriver

when i lunch new compute node with below localrc
Compute Node localrc
==============================================
# -*- mode: shell-script -*-

# Setting OFFLINE to True switches off download of packages.
OFFLINE=False

# Setting RECLONE=yes will make stack.sh freshen each repo on each run.
RECLONE=yes

# Setting USE_TUNNELS=True will make OVS plugin run in tunnel mode.
USE_TUNNELS=True

# Destination path for installation of the OpenStack components.
# There is no need to specify it unless you want the code in
# some particular location (like in a directory shared by all VMs).
#DEST=/opt/stack

# Running DevStack with multiple hosts requires a custom localrc file
# for each host. The master is the same as a single host in installation
# with MULTI_HOST=True.
MULTI_HOST=True

# Run these services.
#ENABLED_SERVICES=n-cpu,n-net,n-api
#ENABLED_SERVICES=rabbit,n-cpu,quantum,q-agt,g-api
ENABLED_SERVICES=n-cpu,quantum,q-agt

# Use the OpenVSwitch plugin.
Q_PLUGIN=openvswitch

# The ComputeNode VM has direct access to the internet only to
# simply updates using apt-get, git, etc.
# eth0 - Convenience interface towards Internet
# eth1 - interface to management network
# eth2 - interface for tenant VM traffic

# The management network where the Devstack machines reside.
# ALWAYS VERIFY IT BEFORE RUNNING stack.sh!
MY_NET='172.16.1.0/24'

# The IP addresses of the controller node.
# The Internet facing IP address. Set it to IP number rather than name
# since we don't have names in DNS.
SERVICE_HOST=`awk '/ControllerPublic/ { print $1; } ' /etc/hosts`
# The management network facing IP address.
CONTROLLER_HOST=Controller

#######################################################################
# OpenVSwitch plugin settings
# ---------------------------

if [ "$USE_TUNNELS" == "True" ]; then
    # Set OVS plugin to (GRE) tunnel mode.
    ENABLE_TENANT_TUNNELS=True
    OVS_ENABLE_TUNNELING=True
    TENANT_TUNNEL_RANGE=500:1500
    ENABLE_TENANT_VLANS=False

    # The interface and bridge for tenant VM traffic.
    #GUEST_INTERFACE_DEFAULT=eth2

    tunnel_flag="-g"
else
    # Set OVS plugin to VLAN mode.
    ENABLE_TENANT_TUNNELS=False
    OVS_ENABLE_TUNNELING=False
    ENABLE_TENANT_VLANS=True
    TENANT_VLAN_RANGE=500:1500
    PHYSICAL_NETWORK=default

    # The interface for tenant VM traffic.
    #GUEST_INTERFACE_DEFAULT=eth2
    OVS_PHYSICAL_BRIDGE=br-eth2
    OVS_BRIDGE_MAPPINGS=$PHYSICAL_NETWORK:$OVS_PHYSICAL_BRIDGE

    tunnel_flag=""
fi

# Cleanup old stuff that may remain. We only do this when stack.sh is
# the executed script.
scriptname=`basename $0`
if [ $scriptname == "stack.sh" ]; then
    sudo ~/bin/quantum_cleanup.sh
fi

# Setup the interfaces and bridges that the 'stack.sh' script won't.
# Again we only do this if stack.sh is the executed script.
if [ $scriptname == "stack.sh" ]; then
    host=`hostname`
    cidr=`grep $host /etc/hosts | awk '/172.16.2/ { print $1 }'`'/24'
    sudo ~/bin/quantum_if_and_br_setup.sh $tunnel_flag -i eth2 -n $cidr
fi

# End of OpenVSwitch plugin settings.
########################################################################

# ``HOST_IP`` is not set by default.
# This is the management network IP address of this ComputeNode.
HOST_IP=`~/bin/hostitfc -i -c $MY_NET`

# Settings to get NoVNC to work.
VNCSERVER_LISTEN=$HOST_IP
VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP

# Some network address settings relevant for my configuration.
FLAT_INTERFACE=`~/bin/hostitfc -n -c $MY_NET`
FIXED_RANGE=10.0.3.0/24
NETWORK_GATEWAY=10.0.3.1
FIXED_NETWORK_SIZE=256
FLOATING_RANGE=172.16.6.32/27

# Logging
# -------

# By default ``stack.sh`` output only goes to the terminal where it runs. It can
# be configured to additionally log to a file by setting ``LOGFILE`` to the full
# path of the destination log file. A timestamp will be appended to the given name.
LOGFILE=~/devstack/stack.sh.log

MYSQL_PASSWORD=simple
MYSQL_USER=stacker

RABBIT_PASSWORD=simple
SERVICE_TOKEN=simple
SERVICE_PASSWORD=simple
ADMIN_PASSWORD=simple
SERVICE_TOKEN=simple

MYSQL_HOST=$CONTROLLER_HOST
RABBIT_HOST=$CONTROLLER_HOST
KEYSTONE_AUTH_HOST=$CONTROLLER_HOST
GLANCE_HOSTPORT=$CONTROLLER_HOST:9292
Q_HOST=$CONTROLLER_HOST
#SYSLOG_HOST=$CONTROLLER_HOST

# Type of virtualization to use. Options: kvm, lxc, qemu
LIBVIRT_TYPE=kvm
# Uncomment this to use LXC virtualization instead of (slower) QEMU.
#LIBVIRT_TYPE=lxc
=================================================

I got below syslog, one thing need notice is when new compute node executing nova-manage db sync.....

14:29 14929 ERROR nova.manager [-] Error during ComputeManager._cleanup_running_deleted_instances: (ProgrammingError) (1146, "Table 'nova.instances' doesn't exist") 'SELECT instances.created_at AS instances_created_at, instances.updated_at AS instances_updated_at, instances.deleted_at AS instances_deleted_at, instances.deleted AS instances_deleted, instances.id AS instances_id, instances.user_id AS instances_user_id, instances.project_id AS instances_project_id, instances.image_ref AS instances_image_ref, instances.kernel_id AS instances_kernel_id, instances.ramdisk_id AS instances_ramdisk_id, instances.hostname AS instances_hostname, instances.launch_index AS instances_launch_index, instances.key_name AS instances_key_name, instances.key_data AS instances_key_data, instances.power_state AS instances_power_state, instances.vm_state AS instances_vm_state, instances.task_state AS instances_task_state, instances.memory_mb AS instances_memory_mb, instances.vcpus AS instances_vcpus, instances.root_gb AS instances_root_gb, instances.ephemeral_gb AS instances_ephemeral_gb, instances.host AS instances_host, instances.node AS instances_node, instances.instance_type_id AS instances_instance_type_id, instances.user_data AS instances_user_data, instances.reservation_id AS instances_reservation_id, instances.scheduled_at AS instances_scheduled_at, instances.launched_at AS instances_launched_at, instances.terminated_at AS instances_terminated_at, instances.availability_zone AS instances_availability_zone, instances.display_name AS instances_display_name, instances.display_description AS instances_display_description, instances.launched_on AS instances_launched_on, instances.locked AS instances_locked, instances.os_type AS instances_os_type, instances.architecture AS instances_architecture, instances.vm_mode AS instances_vm_mode, instances.uuid AS instances_uuid, instances.root_device_name AS instances_root_device_name, instances.default_ephemeral_device AS instances_default_ephemeral_device, instances.default

this result in all tables in nova databases are gone.

mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| cinder |
| glance |
| keystone |
| mysql |
| nova |
| ovs_quantum |
| performance_schema |
| test |
+--------------------+
9 rows in set (0.00 sec)

mysql> use nova;
Database changed
mysql> show tables;
Empty set (0.00 sec)

mysql>

[Launchpad Janitor (janitor)]

[Expired for devstack because there has been no activity for 60 days.]