Juniper Openstack

  1. Bug #1777561
  2. Activity log

Activity log for bug #1777561

Date Who What changed Old value New value Message
2018-06-19 03:48:11 Venkatesh Velpula bug added bug
2018-06-19 03:56:44 Venkatesh Velpula description [root@testbed-1-vm4 ~]# docker ps | grep config_api c75b52476e9a 10.204.217.152:5000/contrail-controller-config-api:ocata-master-144 "/entrypoint.sh /u..." 16 hours ago config_api_1 [root@testbed-1-vm4 ~]# docker restart c75b52476e9a docker ps | grep config_api c75b52476e9a 10.204.217.152:5000/contrail-controller-config-api:ocata-master-144 "/entrypoint.sh /u..." 16 hours ago Restarting (1) About a minute ago config_api_1 == Contrail control == control: active nodemgr: active named: active dns: active == Contrail kubernetes == kube-manager: active == Contrail database == nodemgr: active zookeeper: active cassandra: active == Contrail analytics == nodemgr: active api: active collector: active query-engine: active == Contrail webui == web: active job: active == Contrail config == api: inactive zookeeper: active svc-monitor: active nodemgr: active device-manager: backup cassandra: active rabbitmq: active schema: active [root@testbed-1-vm4 ~]# contrail-version Package Version Build-ID | Repo | RPM Name -------------------------------------- ------------------------------ ---------------------------------- contrail-lib 5.1.0-144.el7 @contrail python-contrail 5.1.0-144.el7 @contrail contrail-utils 5.1.0-144.el7 @contrail contrail-kube-manager 5.1.0-144.el7 @contrail contrail-setup 5.1.0-144.el7 @contrail docker logs ============ [root@testbed-1-vm4 ~]# docker logs c75b52476e9a ++ LOG_DIR=/var/log/contrail ++ LOG_LOCAL=1 +++ get_default_nic +++ ip route get 1 +++ grep -o 'dev.*' +++ awk '{print $2}' ++ DEFAULT_IFACE=eth0 +++ get_default_ip ++++ get_default_nic ++++ ip route get 1 ++++ grep -o 'dev.*' ++++ awk '{print $2}' +++ local nic=eth0 +++ get_cidr_for_nic eth0 +++ local nic=eth0 +++ cut -d / -f 1 +++ grep 'inet ' +++ ip addr show dev eth0 +++ awk '{print $2}' +++ head -n 1 ++ DEFAULT_LOCAL_IP=10.204.218.100 +++ uname -n ++ DEFAULT_HOSTNAME=testbed-1-vm4 ++ CLOUD_ORCHESTRATOR=kubernetes ++ AAA_MODE=no-auth ++ AUTH_MODE=noauth ++ [[ kubernetes == \o\p\e\n\s\t\a\c\k ]] ++ CLOUD_ADMIN_ROLE=admin ++ SSL_ENABLE=False ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ CONTROLLER_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ CONFIG_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ CONTROL_NODES=192.168.1.5,192.168.1.8,192.168.1.9 ++ DNS_NODES=192.168.1.5,192.168.1.8,192.168.1.9 ++ CONFIGDB_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ ZOOKEEPER_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ RABBITMQ_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ ANALYTICS_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ ANALYTICSDB_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ ZOOKEEPER_ANALYTICS_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ KAFKA_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ TSN_NODES='[]' ++ KUBERNETES_API_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ CONTROL_INTROSPECT_PORT=8083 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ XMPP_SERVER_PORT=5269 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ RABBITMQ_NODE_PORT=5673 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 ++ ZOOKEEPER_ANALYTICS_PORT=2182 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/pki/ca-trust/source/anchors/contrail_webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/pki/ca-trust/source/anchors/contrail_webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ QUERYENGINE_INTROSPECT_PORT=8091 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ REDIS_SERVER_PORT=6379 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ KAFKA_PORT=9092 ++ METADATA_PROXY_SECRET=c0ntrail123 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ REDIS_SERVER_PASSWORD= ++ EXTERNAL_ROUTERS= ++ BGP_ASN=64512 ++ SUBCLUSTER= ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ OPENSTACK_LBAAS_AUTH= ++ KUBERNETES_LBAAS_AUTH= ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/v3 ++ KEYSTONE_AUTH_HOST=127.0.0.1 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=35357 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ [[ /v3 == \/\v\2\.\0 ]] ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AUTH_PARAMS= ++ [[ noauth == \k\e\y\s\t\o\n\e ]] +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in '"${server_list[@]}"' ++++ echo 10.204.217.194 +++ local server_address=10.204.217.194 +++ extended_server_list+=10.204.217.194:2181, +++ for server in '"${server_list[@]}"' ++++ echo 10.204.218.100 +++ local server_address=10.204.218.100 +++ extended_server_list+=10.204.218.100:2181, +++ for server in '"${server_list[@]}"' ++++ echo 10.204.218.101 +++ local server_address=10.204.218.101 +++ extended_server_list+=10.204.218.101:2181, +++ '[' -n 10.204.217.194:2181,10.204.218.100:2181,10.204.218.101:2181, ']' +++ echo 10.204.217.194:2181,10.204.218.100:2181,10.204.218.101:2181 ++ ZOOKEEPER_SERVERS=10.204.217.194:2181,10.204.218.100:2181,10.204.218.101:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= ++ ANALYTICS_API_VIP= ++ CONFIG_API_VIP= ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ XMPP_SSL_ENABLE=False ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_SSL_ENABLE=False ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SANDESH_SSL_ENABLE=False ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=False ++ RABBITMQ_SSL_VER= ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ AGENT_MODE=kernel ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ HUGE_PAGES=1024 ++ HUGE_PAGES_DIR=/dev/hugepages ++ DPDK_MEM_PER_SOCKET=1024 ++ VHOST_CONFIG_DIR=/etc/sysconfig/network-scripts ++ TSN_EVPN_MODE=False ++ VROUTER_CRYPT_INTERFACE=crypt0 ++ VROUTER_DECRYPT_INTERFACE=decrypt0 ++ VROUTER_DECRYPT_KEY=15 ++ DIST_SNAT_PROTO_PORT_LIST= ++ FLOW_EXPORT_RATE=0 ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return ++ get_listen_ip_for_node CONFIG +++ find_my_ip_and_order_for_node CONFIG +++ local server_typ=CONFIG_NODES +++ local server_list= +++ IFS=, +++ read -ra server_list +++ cut -d ' ' -f 1 ++++ get_local_ips ++++ tr '\n' , ++++ cat /proc/net/fib_trie ++++ awk '/32 host/ { print f } {f=$2}' +++ local local_ips=,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,, +++ local ord=1 +++ for server in '"${server_list[@]}"' +++ [[ ,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,, =~ ,10\.204\.217\.194, ]] +++ (( ord+=1 )) +++ for server in '"${server_list[@]}"' +++ [[ ,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,, =~ ,10\.204\.218\.100, ]] +++ echo 10.204.218.100 2 +++ return ++ local ip=10.204.218.100 ++ [[ -z 10.204.218.100 ]] ++ echo 10.204.218.100 + host_ip=10.204.218.100 ++ sed 's/,/ /g' ++ echo 10.204.217.194:9161 10.204.218.100:9161 10.204.218.101:9161 + cassandra_server_list='10.204.217.194:9161 10.204.218.100:9161 10.204.218.101:9161' + cat + add_ini_params_from_env API /etc/contrail/contrail-api.conf + local service_name=API + local cfg_path=/etc/contrail/contrail-api.conf + local delim=__ ++ set -o posix ++ set ++ sort ++ grep '^API__.*__.*=.*$' ++ cut -d = -f 1 ++ sed 's/^API__//g' + local vars= + local section= + set_third_party_auth_config + [[ noauth != \k\e\y\s\t\o\n\e ]] + return + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + [[ noauth == \k\e\y\s\t\o\n\e ]] + cat + mv /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + exec /usr/bin/python /usr/bin/contrail-api --conf_file /etc/contrail/contrail-api.conf --conf_file /etc/contrail/contrail-keystone-auth.conf --worker_id 0 06/18/2018 10:57:10 AM [contrail-api] [INFO]: SANDESH: CONNECT TO COLLECTOR: True INFO:contrail-api:SANDESH: CONNECT TO COLLECTOR: True 06/18/2018 10:57:10 AM [contrail-api] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] INFO:contrail-api:SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 06/18/2018 10:57:10 AM [contrail-api] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/contrail-api.log] INFO:contrail-api:SANDESH: Logging: FILE: [None] -> [/var/log/contrail/contrail-api.log] INFO:contrail-api:SANDESH: Logging: SYSLOG: [None] -> [LOG_LOCAL0] INFO:contrail-api:SANDESH: Trace: PRINT: [None] -> [False] INFO:contrail-api:SANDESH: Flow Logging: [None] -> [False] WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: Connecting to zookeeper on 10.204.217.194:2181,10.204.218.100:2181,10.204.218.101:2181 DEBUG:contrail-api:Add uve <testbed-1-vm4, 1> in the [ObjectConfigNode:NodeStatusUVE] map ERROR:contrail-api:Cannot write http_port 8084 to /tmp/contrail-api.0.http_port ERROR:contrail-api:Starting Introspect on HTTP Port 8084 INFO:contrail-api:Processing event[EvStart] in state[none] DEBUG:contrail-api:Add uve <testbed-1-vm4:Config:contrail-api:0, 1> in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-api:Sandesh Client: Event[EvStart] => State[none] -> State[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Initializing description = >>, ] description = Zookeeper:Zookeeper[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Initializing description = >>, << type = Collector name = server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Zookeeper:Zookeeper[], Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = testbed-1-vm4:Config:contrail-api:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8084 start_time = 1529319430623654 collector_name = collector_ip = collector_list = [ 10.204.218.100:8086, 10.204.218.101:8086, 10.204.217.194:8086, ] >> sm_queue_count = 2 max_sm_queue_count = 2 >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:api-0:Connecting to 10.204.218.100:2181 DEBUG:api-0:Sending request(xid=None): Connect(protocol_version=0, last_zxid_seen=0, time_out=400000, session_id=0, passwd='\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', read_only=None) INFO:api-0:Zookeeper connection established, state: CONNECTED INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:api-0:Sending request(xid=1): GetChildren(path='/id/virtual-networks', watcher=None) DEBUG:api-0:Received error(xid=1) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 4862 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=2): GetChildren(path='/id/security-groups/id', watcher=None) DEBUG:api-0:Received response(xid=2): [u'0000000000'] ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 2204 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=3): GetData(path='/id/security-groups/id/0000000000', watcher=None) DEBUG:api-0:Received response(xid=3): ('__reserved__', ZnodeStat(czxid=4294967312, mzxid=4294967312, ctime=1529319421602, mtime=1529319421602, version=0, cversion=0, aversion=0, ephemeralOwner=0, dataLength=12, numChildren=0, pzxid=4294967312)) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = READ application = ZK response_time_in_usec = 2353 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=4): Create(path='/id/security-groups/id/0000000000', data='__reserved__', acl=[ACL(perms=31, acl_list=['ALL'], id=Id(scheme='world', id='anyone'))], flags=0) DEBUG:api-0:Received error(xid=4) NodeExistsError((), {}) DEBUG:api-0:Sending request(xid=5): GetData(path='/id/security-groups/id/0000000000', watcher=None) DEBUG:api-0:Received response(xid=5): ('__reserved__', ZnodeStat(czxid=4294967312, mzxid=4294967312, ctime=1529319421602, mtime=1529319421602, version=0, cversion=0, aversion=0, ephemeralOwner=0, dataLength=12, numChildren=0, pzxid=4294967312)) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = READ application = ZK response_time_in_usec = 2064 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = CREATE application = ZK response_time_in_usec = 2586 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=6): GetChildren(path='/id/tags/types', watcher=None) DEBUG:api-0:Received error(xid=6) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 2036 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=7): GetChildren(path='/id/tags/values/tier', watcher=None) DEBUG:api-0:Received error(xid=7) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 2074 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=8): GetChildren(path='/id/tags/values/application', watcher=None) DEBUG:api-0:Received error(xid=8) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 2012 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=9): GetChildren(path='/id/tags/values/deployment', watcher=None) DEBUG:api-0:Received error(xid=9) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 1976 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=10): GetChildren(path='/id/tags/values/site', watcher=None) DEBUG:api-0:Received error(xid=10) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 1876 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=11): GetChildren(path='/id/tags/values/label', watcher=None) DEBUG:api-0:Received error(xid=11) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 1968 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=12): Exists(path='/api-server-election', watcher=None) DEBUG:api-0:Received response(xid=12): ZnodeStat(czxid=4294967313, mzxid=4294967313, ctime=1529319421740, mtime=1529319421740, version=0, cversion=2, aversion=0, ephemeralOwner=0, dataLength=0, numChildren=2, pzxid=4294967326) DEBUG:api-0:Sending request(xid=13): Create(path='/api-server-election/675705c867cd4117ab5c516f2ffea22a__lock__', data='1', acl=[ACL(perms=31, acl_list=['ALL'], id=Id(scheme='world', id='anyone'))], flags=3) DEBUG:api-0:Received response(xid=13): u'/api-server-election/675705c867cd4117ab5c516f2ffea22a__lock__0000000002' DEBUG:api-0:Sending request(xid=14): GetChildren(path='/api-server-election', watcher=None) DEBUG:api-0:Received response(xid=14): [u'675705c867cd4117ab5c516f2ffea22a__lock__0000000002', u'8f2d6f9d00e24d5b8bb0227cb0d48f08__lock__0000000000', u'f1fdea61d5ce42f99a3907e2914ef121__lock__0000000001'] DEBUG:api-0:Sending request(xid=15): Exists(path=u'/api-server-election/f1fdea61d5ce42f99a3907e2914ef121__lock__0000000001', watcher=<bound method Lock._watch_predecessor of <kazoo.recipe.lock.Lock object at 0x7f25c1cb3e10>>) DEBUG:api-0:Received response(xid=15): ZnodeStat(czxid=4294967326, mzxid=4294967326, ctime=1529319430066, mtime=1529319430066, version=0, cversion=0, aversion=0, ephemeralOwner=172283047723597827, dataLength=1, numChildren=0, pzxid=4294967326) INFO:contrail-api:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-api:Session Event: TCP Connect Fail INFO:contrail-api:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.100:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-api:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-api:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.100:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = testbed-1-vm4:Config:contrail-api:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8084 start_time = 1529319430623654 collector_name = collector_ip = 10.204.218.100:8086 collector_list = [ 10.204.218.100:8086, 10.204.218.101:8086, 10.204.217.194:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 3 >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:api-0:Received EVENT: Watch(type=2, state=3, path=u'/api-server-election/f1fdea61d5ce42f99a3907e2914ef121__lock__0000000001') DEBUG:api-0:Sending request(xid=16): GetChildren(path='/api-server-election', watcher=None) DEBUG:api-0:Received response(xid=16): [u'675705c867cd4117ab5c516f2ffea22a__lock__0000000002'] WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: Connecting to database on ['10.204.217.194:9161', '10.204.218.100:9161', '10.204.218.101:9161'] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.100:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Initializing description = >>, ] description = Collector, Database:Cassandra[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:__default__ [SYS_INFO]: VncApiInfo: Info! InvalidRequestException(why='Cannot add already existing table "useragent_keyval_table" to keyspace "useragent"') INFO:contrail-api:__default__ [SYS_INFO]: VncApiInfo: Info! InvalidRequestException(why='Cannot add already existing table "obj_shared_table" to keyspace "config_db_uuid"') INFO:contrail-api:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-api:Session Event: TCP Connect Fail INFO:contrail-api:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Initializing description = >>, ] description = Collector, Database:Cassandra[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-api:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-api:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Initializing description = >>, ] description = Collector, Database:Cassandra[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = testbed-1-vm4:Config:contrail-api:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8084 start_time = 1529319430623654 collector_name = collector_ip = 10.204.218.101:8086 collector_list = [ 10.204.218.100:8086, 10.204.218.101:8086, 10.204.217.194:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 3 >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:__default__ [SYS_INFO]: VncApiInfo: Info! InvalidRequestException(why='Cannot add already existing table "obj_uuid_table" to keyspace "config_db_uuid"') INFO:contrail-api:__default__ [SYS_INFO]: VncApiInfo: Info! InvalidRequestException(why='Cannot add already existing table "obj_fq_name_table" to keyspace "config_db_uuid"') WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: Cassandra connection ESTABLISHED INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Up description = >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:api-0:Sending request(xid=17): Delete(path=u'/api-server-election/675705c867cd4117ab5c516f2ffea22a__lock__0000000002', version=-1) DEBUG:api-0:Received response(xid=17): True WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: Initializing RabbitMQ connection, urls ['pyamqp://guest:guest@10.204.217.194:5673//', 'pyamqp://guest:guest@10.204.218.100:5673//', 'pyamqp://guest:guest@10.204.218.101:5673//'] WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: RabbitMQ connection down INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Up description = >>, << type = Database name = RabbitMQ server_addrs = [ 10.204.217.194:5673, 10.204.218.100:5673, 10.204.218.101:5673, ] status = Initializing description = >>, ] description = Collector, Database:RabbitMQ[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Up description = >>, << type = Database name = RabbitMQ server_addrs = [ 10.204.217.194:5673, 10.204.218.100:5673, 10.204.218.101:5673, ] status = Down description = >>, ] description = Collector, Database:RabbitMQ[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: RabbitMQ connection ESTABLISHED <Connection: amqp://guest:**@10.204.217.194:5673// at 0x7f25c0fedb10> INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Up description = >>, << type = Database name = RabbitMQ server_addrs = [ 10.204.217.194:5673, 10.204.218.100:5673, 10.204.218.101:5673, ] status = Up description = >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 22142 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> <pre>Traceback (most recent call last): File "/usr/bin/contrail-api", line 9, in &lt;module> load_entry_point('contrail-api-server==0.1dev', 'console_scripts', 'contrail-api')() File "/usr/lib/python2.7/site-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 4721, in server_main main(args_str, VncApiServer(args_str)) File "/usr/lib/python2.7/site-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 1856, in __init__ self._db_init_entries() File "/usr/lib/python2.7/site-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 3207, in _db_init_entries autonomous_system=64512, config_version=CONFIG_VERSION)) File "/usr/lib/python2.7/site-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 3493, in create_singleton_entry cass_uuid = self._db_conn._object_db.fq_name_to_uuid(obj_type, fq_name) File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 514, in wrapper return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 1427, in fq_name_to_uuid finish=fq_name_str + ';') File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 201, in get finish=finish) File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 222, in multiget column_count=column_count) File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 514, in wrapper return func(*args, **kwargs) File "/usr/lib/python2.7/site-packages/pycassa/columnfamily.py", line 772, in multiget packed_keys[offset:offset + buffer_size], cp, sp, consistency) File "/usr/lib/python2.7/site-packages/pycassa/pool.py", line 577, in execute return getattr(conn, f)(*args, **kwargs) File "/usr/lib/python2.7/site-packages/pycassa/pool.py", line 127, in new_f result = f(self, *args, **kwargs) File "/usr/lib/python2.7/site-packages/pycassa/cassandra/Cassandra.py", line 669, in multiget_slice return self.recv_multiget_slice() File "/usr/lib/python2.7/site-packages/pycassa/cassandra/Cassandra.py", line 695, in recv_multiget_slice raise result.ire InvalidRequestException: InvalidRequestException(why='unconfigured table obj_fq_name_table') Build :5.1.0-144 Deployment :Ansible_deploer HOST OS: CENTOS7.5 ======================= Topology ========= instances: vrouter +k8s_node: ip: 10.204.217.197 ip: 10.204.217.198 config +control+k8s_master+kubemanager: ip: 10.204.218.100 ip: 10.204.218.101 ip: 10.204.217.194 =========================================== [root@testbed-1-vm4 ~]# docker ps | grep config_api c75b52476e9a 10.204.217.152:5000/contrail-controller-config-api:ocata-master-144 "/entrypoint.sh /u..." 16 hours ago config_api_1 [root@testbed-1-vm4 ~]# docker restart c75b52476e9a docker ps | grep config_api c75b52476e9a 10.204.217.152:5000/contrail-controller-config-api:ocata-master-144 "/entrypoint.sh /u..." 16 hours ago Restarting (1) About a minute ago config_api_1 == Contrail control == control: active nodemgr: active named: active dns: active == Contrail kubernetes == kube-manager: active == Contrail database == nodemgr: active zookeeper: active cassandra: active == Contrail analytics == nodemgr: active api: active collector: active query-engine: active == Contrail webui == web: active job: active == Contrail config == api: inactive zookeeper: active svc-monitor: active nodemgr: active device-manager: backup cassandra: active rabbitmq: active schema: active [root@testbed-1-vm4 ~]#  contrail-version Package Version Build-ID | Repo | RPM Name -------------------------------------- ------------------------------ ---------------------------------- contrail-lib 5.1.0-144.el7 @contrail python-contrail 5.1.0-144.el7 @contrail contrail-utils 5.1.0-144.el7 @contrail contrail-kube-manager 5.1.0-144.el7 @contrail contrail-setup 5.1.0-144.el7 @contrail docker logs ============ [root@testbed-1-vm4 ~]# docker logs c75b52476e9a ++ LOG_DIR=/var/log/contrail ++ LOG_LOCAL=1 +++ get_default_nic +++ ip route get 1 +++ grep -o 'dev.*' +++ awk '{print $2}' ++ DEFAULT_IFACE=eth0 +++ get_default_ip ++++ get_default_nic ++++ ip route get 1 ++++ grep -o 'dev.*' ++++ awk '{print $2}' +++ local nic=eth0 +++ get_cidr_for_nic eth0 +++ local nic=eth0 +++ cut -d / -f 1 +++ grep 'inet ' +++ ip addr show dev eth0 +++ awk '{print $2}' +++ head -n 1 ++ DEFAULT_LOCAL_IP=10.204.218.100 +++ uname -n ++ DEFAULT_HOSTNAME=testbed-1-vm4 ++ CLOUD_ORCHESTRATOR=kubernetes ++ AAA_MODE=no-auth ++ AUTH_MODE=noauth ++ [[ kubernetes == \o\p\e\n\s\t\a\c\k ]] ++ CLOUD_ADMIN_ROLE=admin ++ SSL_ENABLE=False ++ SSL_INSECURE=True ++ SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SERVER_CA_KEYFILE=/etc/contrail/ssl/private/ca-key.pem ++ CONTROLLER_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ CONFIG_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ CONTROL_NODES=192.168.1.5,192.168.1.8,192.168.1.9 ++ DNS_NODES=192.168.1.5,192.168.1.8,192.168.1.9 ++ CONFIGDB_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ ZOOKEEPER_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ RABBITMQ_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ ANALYTICS_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ ANALYTICSDB_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ ZOOKEEPER_ANALYTICS_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ KAFKA_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ TSN_NODES='[]' ++ KUBERNETES_API_NODES=10.204.217.194,10.204.218.100,10.204.218.101 ++ CONTROL_INTROSPECT_PORT=8083 ++ BGP_PORT=179 ++ BGP_AUTO_MESH=true ++ XMPP_SERVER_PORT=5269 ++ DNS_SERVER_PORT=53 ++ DNS_INTROSPECT_PORT=8092 ++ CONFIG_API_PORT=8082 ++ CONFIG_API_INTROSPECT_PORT=8084 ++ RABBITMQ_NODE_PORT=5673 ++ CONFIGDB_PORT=9161 ++ CONFIGDB_CQL_PORT=9041 ++ ZOOKEEPER_PORT=2181 ++ ZOOKEEPER_PORTS=2888:3888 ++ ZOOKEEPER_ANALYTICS_PORT=2182 ++ WEBUI_JOB_SERVER_PORT=3000 ++ KUE_UI_PORT=3002 ++ WEBUI_HTTP_LISTEN_PORT=8180 ++ WEBUI_HTTPS_LISTEN_PORT=8143 ++ WEBUI_SSL_KEY_FILE=/etc/pki/ca-trust/source/anchors/contrail_webui_ssl/cs-key.pem ++ WEBUI_SSL_CERT_FILE=/etc/pki/ca-trust/source/anchors/contrail_webui_ssl/cs-cert.pem ++ WEBUI_SSL_CIPHERS=ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:AES256-SHA ++ ANALYTICS_API_PORT=8081 ++ ANALYTICS_API_INTROSPECT_PORT=8090 ++ COLLECTOR_PORT=8086 ++ COLLECTOR_INTROSPECT_PORT=8089 ++ COLLECTOR_SYSLOG_PORT=514 ++ COLLECTOR_SFLOW_PORT=6343 ++ COLLECTOR_IPFIX_PORT=4739 ++ COLLECTOR_PROTOBUF_PORT=3333 ++ COLLECTOR_STRUCTURED_SYSLOG_PORT=3514 ++ ALARMGEN_INTROSPECT_PORT=5995 ++ QUERYENGINE_INTROSPECT_PORT=8091 ++ SNMPCOLLECTOR_INTROSPECT_PORT=5920 ++ TOPOLOGY_INTROSPECT_PORT=5921 ++ REDIS_SERVER_PORT=6379 ++ ANALYTICSDB_PORT=9160 ++ ANALYTICSDB_CQL_PORT=9042 ++ CASSANDRA_PORT=9160 ++ CASSANDRA_CQL_PORT=9042 ++ CASSANDRA_SSL_STORAGE_PORT=7011 ++ CASSANDRA_STORAGE_PORT=7010 ++ CASSANDRA_JMX_LOCAL_PORT=7200 ++ KAFKA_PORT=9092 ++ METADATA_PROXY_SECRET=c0ntrail123 ++ ENCAP_PRIORITY=MPLSoUDP,MPLSoGRE,VXLAN ++ REDIS_SERVER_PASSWORD= ++ EXTERNAL_ROUTERS= ++ BGP_ASN=64512 ++ SUBCLUSTER= ++ RNDC_KEY=xvysmOR8lnUQRBcunkC6vg== ++ OPENSTACK_LBAAS_AUTH= ++ KUBERNETES_LBAAS_AUTH= ++ KEYSTONE_AUTH_ADMIN_TENANT=admin ++ KEYSTONE_AUTH_ADMIN_USER=admin ++ KEYSTONE_AUTH_ADMIN_PASSWORD=contrail123 ++ KEYSTONE_AUTH_PROJECT_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_USER_DOMAIN_NAME=Default ++ KEYSTONE_AUTH_REGION_NAME=RegionOne ++ KEYSTONE_AUTH_URL_VERSION=/v3 ++ KEYSTONE_AUTH_HOST=127.0.0.1 ++ KEYSTONE_AUTH_PROTO=http ++ KEYSTONE_AUTH_ADMIN_PORT=35357 ++ KEYSTONE_AUTH_PUBLIC_PORT=5000 ++ KEYSTONE_AUTH_URL_TOKENS=/v3/auth/tokens ++ [[ /v3 == \/\v\2\.\0 ]] ++ KEYSTONE_AUTH_INSECURE=True ++ KEYSTONE_AUTH_CERTFILE= ++ KEYSTONE_AUTH_KEYFILE= ++ KEYSTONE_AUTH_CA_CERTFILE= ++ BARBICAN_USER=barbican ++ BARBICAN_PASSWORD=contrail123 ++ AUTH_PARAMS= ++ [[ noauth == \k\e\y\s\t\o\n\e ]] +++ get_server_list CONFIG ':8082 ' +++ local server_typ=CONFIG_NODES +++ local 'port_with_delim=:8082 ' +++ local server_list= +++ IFS=, +++ read -ra server_list +++ local extended_server_list= +++ for server in '"${server_list[@]}"' ++++ echo 10.204.217.194 +++ local server_address=10.204.217.194 +++ extended_server_list+=10.204.217.194:2181, +++ for server in '"${server_list[@]}"' ++++ echo 10.204.218.100 +++ local server_address=10.204.218.100 +++ extended_server_list+=10.204.218.100:2181, +++ for server in '"${server_list[@]}"' ++++ echo 10.204.218.101 +++ local server_address=10.204.218.101 +++ extended_server_list+=10.204.218.101:2181, +++ '[' -n 10.204.217.194:2181,10.204.218.100:2181,10.204.218.101:2181, ']' +++ echo 10.204.217.194:2181,10.204.218.100:2181,10.204.218.101:2181 ++ ZOOKEEPER_SERVERS=10.204.217.194:2181,10.204.218.100:2181,10.204.218.101:2181 +++ get_server_list ZOOKEEPER ':2181 ' +++ local server_typ=ZOOKEEPER_NODES +++ local 'port_with_delim=:2181 ' +++ local server_list= ++ ANALYTICS_API_VIP= ++ CONFIG_API_VIP= ++ LINKLOCAL_SERVICE_PORT=80 ++ LINKLOCAL_SERVICE_NAME=metadata ++ LINKLOCAL_SERVICE_IP=169.254.169.254 ++ IPFABRIC_SERVICE_PORT=8775 ++ XMPP_SSL_ENABLE=False ++ XMPP_SERVER_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ XMPP_SERVER_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ XMPP_SERVER_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ INTROSPECT_SSL_ENABLE=False ++ INTROSPECT_SSL_INSECURE=True ++ INTROSPECT_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ INTROSPECT_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ INTROSPECT_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ SANDESH_SSL_ENABLE=False ++ SANDESH_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ SANDESH_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ SANDESH_CA_CERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ read -r -d '' sandesh_client_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ xmpp_certs_config= ++ METADATA_SSL_ENABLE=false ++ METADATA_SSL_CERTFILE= ++ METADATA_SSL_KEYFILE= ++ METADATA_SSL_CA_CERTFILE= ++ METADATA_SSL_CERT_TYPE= ++ RABBITMQ_VHOST=/ ++ RABBITMQ_USER=guest ++ RABBITMQ_PASSWORD=guest ++ RABBITMQ_USE_SSL=False ++ RABBITMQ_SSL_VER= ++ RABBITMQ_CLIENT_SSL_CERTFILE=/etc/contrail/ssl/certs/server.pem ++ RABBITMQ_CLIENT_SSL_KEYFILE=/etc/contrail/ssl/private/server-privkey.pem ++ RABBITMQ_CLIENT_SSL_CACERTFILE=/etc/contrail/ssl/certs/ca-cert.pem ++ read -r -d '' rabbitmq_config ++ true ++ read -r -d '' rabbit_config ++ true ++ is_enabled False ++ local val=false ++ [[ false == \t\r\u\e ]] ++ [[ false == \y\e\s ]] ++ [[ false == \e\n\a\b\l\e\d ]] ++ AGENT_MODE=kernel ++ DPDK_UIO_DRIVER=uio_pci_generic ++ CPU_CORE_MASK=0x01 ++ HUGE_PAGES=1024 ++ HUGE_PAGES_DIR=/dev/hugepages ++ DPDK_MEM_PER_SOCKET=1024 ++ VHOST_CONFIG_DIR=/etc/sysconfig/network-scripts ++ TSN_EVPN_MODE=False ++ VROUTER_CRYPT_INTERFACE=crypt0 ++ VROUTER_DECRYPT_INTERFACE=decrypt0 ++ VROUTER_DECRYPT_KEY=15 ++ DIST_SNAT_PROTO_PORT_LIST= ++ FLOW_EXPORT_RATE=0 ++ FABRIC_SNAT_HASH_TABLE_SIZE=4096 ++ PRIORITY_ID= ++ PRIORITY_BANDWIDTH= ++ PRIORITY_SCHEDULING= ++ QOS_QUEUE_ID= ++ QOS_LOGICAL_QUEUES= ++ QOS_DEF_HW_QUEUE=False ++ PRIORITY_TAGGING=True + pre_start_init + wait_certs_if_ssl_enabled + is_ssl_enabled + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + is_enabled False + local val=false + [[ false == \t\r\u\e ]] + [[ false == \y\e\s ]] + [[ false == \e\n\a\b\l\e\d ]] + return ++ get_listen_ip_for_node CONFIG +++ find_my_ip_and_order_for_node CONFIG +++ local server_typ=CONFIG_NODES +++ local server_list= +++ IFS=, +++ read -ra server_list +++ cut -d ' ' -f 1 ++++ get_local_ips ++++ tr '\n' , ++++ cat /proc/net/fib_trie ++++ awk '/32 host/ { print f } {f=$2}' +++ local local_ips=,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,, +++ local ord=1 +++ for server in '"${server_list[@]}"' +++ [[ ,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,, =~ ,10\.204\.217\.194, ]] +++ (( ord+=1 )) +++ for server in '"${server_list[@]}"' +++ [[ ,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,10.204.218.100,127.0.0.1,172.17.0.1,192.168.1.8,, =~ ,10\.204\.218\.100, ]] +++ echo 10.204.218.100 2 +++ return ++ local ip=10.204.218.100 ++ [[ -z 10.204.218.100 ]] ++ echo 10.204.218.100 + host_ip=10.204.218.100 ++ sed 's/,/ /g' ++ echo 10.204.217.194:9161 10.204.218.100:9161 10.204.218.101:9161 + cassandra_server_list='10.204.217.194:9161 10.204.218.100:9161 10.204.218.101:9161' + cat + add_ini_params_from_env API /etc/contrail/contrail-api.conf + local service_name=API + local cfg_path=/etc/contrail/contrail-api.conf + local delim=__ ++ set -o posix ++ set ++ sort ++ grep '^API__.*__.*=.*$' ++ cut -d = -f 1 ++ sed 's/^API__//g' + local vars= + local section= + set_third_party_auth_config + [[ noauth != \k\e\y\s\t\o\n\e ]] + return + set_vnc_api_lib_ini + local tmp_file=/etc/contrail/vnc_api_lib.ini.tmp + cat + [[ noauth == \k\e\y\s\t\o\n\e ]] + cat + mv /etc/contrail/vnc_api_lib.ini.tmp /etc/contrail/vnc_api_lib.ini + exec /usr/bin/python /usr/bin/contrail-api --conf_file /etc/contrail/contrail-api.conf --conf_file /etc/contrail/contrail-keystone-auth.conf --worker_id 0 06/18/2018 10:57:10 AM [contrail-api] [INFO]: SANDESH: CONNECT TO COLLECTOR: True INFO:contrail-api:SANDESH: CONNECT TO COLLECTOR: True 06/18/2018 10:57:10 AM [contrail-api] [INFO]: SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] INFO:contrail-api:SANDESH: Logging: LEVEL: [SYS_INFO] -> [SYS_DEBUG] 06/18/2018 10:57:10 AM [contrail-api] [INFO]: SANDESH: Logging: FILE: [None] -> [/var/log/contrail/contrail-api.log] INFO:contrail-api:SANDESH: Logging: FILE: [None] -> [/var/log/contrail/contrail-api.log] INFO:contrail-api:SANDESH: Logging: SYSLOG: [None] -> [LOG_LOCAL0] INFO:contrail-api:SANDESH: Trace: PRINT: [None] -> [False] INFO:contrail-api:SANDESH: Flow Logging: [None] -> [False] WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: Connecting to zookeeper on 10.204.217.194:2181,10.204.218.100:2181,10.204.218.101:2181 DEBUG:contrail-api:Add uve <testbed-1-vm4, 1> in the [ObjectConfigNode:NodeStatusUVE] map ERROR:contrail-api:Cannot write http_port 8084 to /tmp/contrail-api.0.http_port ERROR:contrail-api:Starting Introspect on HTTP Port 8084 INFO:contrail-api:Processing event[EvStart] in state[none] DEBUG:contrail-api:Add uve <testbed-1-vm4:Config:contrail-api:0, 1> in the [ObjectGeneratorInfo:SandeshModuleClientTrace] map INFO:contrail-api:Sandesh Client: Event[EvStart] => State[none] -> State[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Initializing description = >>, ] description = Zookeeper:Zookeeper[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Initializing description = >>, << type = Collector name = server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Zookeeper:Zookeeper[], Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = testbed-1-vm4:Config:contrail-api:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8084 start_time = 1529319430623654 collector_name = collector_ip = collector_list = [ 10.204.218.100:8086, 10.204.218.101:8086, 10.204.217.194:8086, ] >> sm_queue_count = 2 max_sm_queue_count = 2 >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:api-0:Connecting to 10.204.218.100:2181 DEBUG:api-0:Sending request(xid=None): Connect(protocol_version=0, last_zxid_seen=0, time_out=400000, session_id=0, passwd='\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', read_only=None) INFO:api-0:Zookeeper connection established, state: CONNECTED INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ , ] status = Down description = none to Idle on EvStart >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:api-0:Sending request(xid=1): GetChildren(path='/id/virtual-networks', watcher=None) DEBUG:api-0:Received error(xid=1) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 4862 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=2): GetChildren(path='/id/security-groups/id', watcher=None) DEBUG:api-0:Received response(xid=2): [u'0000000000'] ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 2204 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=3): GetData(path='/id/security-groups/id/0000000000', watcher=None) DEBUG:api-0:Received response(xid=3): ('__reserved__', ZnodeStat(czxid=4294967312, mzxid=4294967312, ctime=1529319421602, mtime=1529319421602, version=0, cversion=0, aversion=0, ephemeralOwner=0, dataLength=12, numChildren=0, pzxid=4294967312)) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = READ application = ZK response_time_in_usec = 2353 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=4): Create(path='/id/security-groups/id/0000000000', data='__reserved__', acl=[ACL(perms=31, acl_list=['ALL'], id=Id(scheme='world', id='anyone'))], flags=0) DEBUG:api-0:Received error(xid=4) NodeExistsError((), {}) DEBUG:api-0:Sending request(xid=5): GetData(path='/id/security-groups/id/0000000000', watcher=None) DEBUG:api-0:Received response(xid=5): ('__reserved__', ZnodeStat(czxid=4294967312, mzxid=4294967312, ctime=1529319421602, mtime=1529319421602, version=0, cversion=0, aversion=0, ephemeralOwner=0, dataLength=12, numChildren=0, pzxid=4294967312)) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = READ application = ZK response_time_in_usec = 2064 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = CREATE application = ZK response_time_in_usec = 2586 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=6): GetChildren(path='/id/tags/types', watcher=None) DEBUG:api-0:Received error(xid=6) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 2036 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=7): GetChildren(path='/id/tags/values/tier', watcher=None) DEBUG:api-0:Received error(xid=7) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 2074 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=8): GetChildren(path='/id/tags/values/application', watcher=None) DEBUG:api-0:Received error(xid=8) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 2012 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=9): GetChildren(path='/id/tags/values/deployment', watcher=None) DEBUG:api-0:Received error(xid=9) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 1976 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=10): GetChildren(path='/id/tags/values/site', watcher=None) DEBUG:api-0:Received error(xid=10) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 1876 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=11): GetChildren(path='/id/tags/values/label', watcher=None) DEBUG:api-0:Received error(xid=11) NoNodeError((), {}) ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = GET_CHILDREN application = ZK response_time_in_usec = 1968 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> DEBUG:api-0:Sending request(xid=12): Exists(path='/api-server-election', watcher=None) DEBUG:api-0:Received response(xid=12): ZnodeStat(czxid=4294967313, mzxid=4294967313, ctime=1529319421740, mtime=1529319421740, version=0, cversion=2, aversion=0, ephemeralOwner=0, dataLength=0, numChildren=2, pzxid=4294967326) DEBUG:api-0:Sending request(xid=13): Create(path='/api-server-election/675705c867cd4117ab5c516f2ffea22a__lock__', data='1', acl=[ACL(perms=31, acl_list=['ALL'], id=Id(scheme='world', id='anyone'))], flags=3) DEBUG:api-0:Received response(xid=13): u'/api-server-election/675705c867cd4117ab5c516f2ffea22a__lock__0000000002' DEBUG:api-0:Sending request(xid=14): GetChildren(path='/api-server-election', watcher=None) DEBUG:api-0:Received response(xid=14): [u'675705c867cd4117ab5c516f2ffea22a__lock__0000000002', u'8f2d6f9d00e24d5b8bb0227cb0d48f08__lock__0000000000', u'f1fdea61d5ce42f99a3907e2914ef121__lock__0000000001'] DEBUG:api-0:Sending request(xid=15): Exists(path=u'/api-server-election/f1fdea61d5ce42f99a3907e2914ef121__lock__0000000001', watcher=<bound method Lock._watch_predecessor of <kazoo.recipe.lock.Lock object at 0x7f25c1cb3e10>>) DEBUG:api-0:Received response(xid=15): ZnodeStat(czxid=4294967326, mzxid=4294967326, ctime=1529319430066, mtime=1529319430066, version=0, cversion=0, aversion=0, ephemeralOwner=172283047723597827, dataLength=1, numChildren=0, pzxid=4294967326) INFO:contrail-api:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-api:Session Event: TCP Connect Fail INFO:contrail-api:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.100:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-api:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-api:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.100:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = testbed-1-vm4:Config:contrail-api:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8084 start_time = 1529319430623654 collector_name = collector_ip = 10.204.218.100:8086 collector_list = [ 10.204.218.100:8086, 10.204.218.101:8086, 10.204.217.194:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 3 >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:api-0:Received EVENT: Watch(type=2, state=3, path=u'/api-server-election/f1fdea61d5ce42f99a3907e2914ef121__lock__0000000001') DEBUG:api-0:Sending request(xid=16): GetChildren(path='/api-server-election', watcher=None) DEBUG:api-0:Received response(xid=16): [u'675705c867cd4117ab5c516f2ffea22a__lock__0000000002'] WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: Connecting to database on ['10.204.217.194:9161', '10.204.218.100:9161', '10.204.218.101:9161'] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.100:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Initializing description = >>, ] description = Collector, Database:Cassandra[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:__default__ [SYS_INFO]: VncApiInfo: Info! InvalidRequestException(why='Cannot add already existing table "useragent_keyval_table" to keyspace "useragent"') INFO:contrail-api:__default__ [SYS_INFO]: VncApiInfo: Info! InvalidRequestException(why='Cannot add already existing table "obj_shared_table" to keyspace "config_db_uuid"') INFO:contrail-api:Processing event[EvIdleHoldTimerExpired] in state[Idle] ERROR:contrail-api:Session Event: TCP Connect Fail INFO:contrail-api:Sandesh Client: Event[EvIdleHoldTimerExpired] => State[Idle] -> State[Connect] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Connect] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Initializing description = Idle to Connect on EvIdleHoldTimerExpired >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Initializing description = >>, ] description = Collector, Database:Cassandra[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Connect] INFO:contrail-api:Processing event[EvTcpConnectFail] in state[Connect] INFO:contrail-api:Sandesh Client: Event[EvTcpConnectFail] => State[Connect] -> State[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Initializing description = >>, ] description = Collector, Database:Cassandra[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] SandeshModuleClientTrace: data = << name = testbed-1-vm4:Config:contrail-api:0 client_info = << status = Idle successful_connections = 0 pid = 1 http_port = 8084 start_time = 1529319430623654 collector_name = collector_ip = 10.204.218.101:8086 collector_list = [ 10.204.218.100:8086, 10.204.218.101:8086, 10.204.217.194:8086, ] >> sm_queue_count = 1 max_sm_queue_count = 3 >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:__default__ [SYS_INFO]: VncApiInfo: Info! InvalidRequestException(why='Cannot add already existing table "obj_uuid_table" to keyspace "config_db_uuid"') INFO:contrail-api:__default__ [SYS_INFO]: VncApiInfo: Info! InvalidRequestException(why='Cannot add already existing table "obj_fq_name_table" to keyspace "config_db_uuid"') WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: Cassandra connection ESTABLISHED INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Up description = >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] DEBUG:api-0:Sending request(xid=17): Delete(path=u'/api-server-election/675705c867cd4117ab5c516f2ffea22a__lock__0000000002', version=-1) DEBUG:api-0:Received response(xid=17): True WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: Initializing RabbitMQ connection, urls ['pyamqp://guest:guest@10.204.217.194:5673//', 'pyamqp://guest:guest@10.204.218.100:5673//', 'pyamqp://guest:guest@10.204.218.101:5673//'] WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: RabbitMQ connection down INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Up description = >>, << type = Database name = RabbitMQ server_addrs = [ 10.204.217.194:5673, 10.204.218.100:5673, 10.204.218.101:5673, ] status = Initializing description = >>, ] description = Collector, Database:RabbitMQ[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Up description = >>, << type = Database name = RabbitMQ server_addrs = [ 10.204.217.194:5673, 10.204.218.100:5673, 10.204.218.101:5673, ] status = Down description = >>, ] description = Collector, Database:RabbitMQ[] connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] WARNING:contrail-api:__default__ [SYS_NOTICE]: VncApiNotice: RabbitMQ connection ESTABLISHED <Connection: amqp://guest:**@10.204.217.194:5673// at 0x7f25c0fedb10> INFO:contrail-api:Processing event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: WrongClientSMState] NodeStatusUVE: data = << name = testbed-1-vm4 process_status = [ << module_id = contrail-api instance_id = 0 state = Non-Functional connection_infos = [ << type = Zookeeper name = Zookeeper server_addrs = [ 10.204.217.194:2181, 10.204.218.100:2181, 10.204.218.101:2181, ] status = Up description = >>, << type = Collector name = server_addrs = [ 10.204.218.101:8086, ] status = Down description = Connect to Idle on EvTcpConnectFail >>, << type = Database name = Cassandra server_addrs = [ 10.204.217.194:9161, 10.204.218.100:9161, 10.204.218.101:9161, ] status = Up description = >>, << type = Database name = RabbitMQ server_addrs = [ 10.204.217.194:5673, 10.204.218.100:5673, 10.204.218.101:5673, ] status = Up description = >>, ] description = Collector connection down >>, ] >> INFO:contrail-api:Discarding event[EvSandeshUVESend] in state[Idle] ERROR:contrail-api:SANDESH: [DROP: NoSession] __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 22142 response_size = 0 identifier = req-1fb03668-c354-4479-af5f-e848f0649288 >> <pre>Traceback (most recent call last):   File "/usr/bin/contrail-api", line 9, in &lt;module>     load_entry_point('contrail-api-server==0.1dev', 'console_scripts', 'contrail-api')()   File "/usr/lib/python2.7/site-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 4721, in server_main     main(args_str, VncApiServer(args_str))   File "/usr/lib/python2.7/site-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 1856, in __init__     self._db_init_entries()   File "/usr/lib/python2.7/site-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 3207, in _db_init_entries     autonomous_system=64512, config_version=CONFIG_VERSION))   File "/usr/lib/python2.7/site-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 3493, in create_singleton_entry     cass_uuid = self._db_conn._object_db.fq_name_to_uuid(obj_type, fq_name)   File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 514, in wrapper     return func(*args, **kwargs)   File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 1427, in fq_name_to_uuid     finish=fq_name_str + ';')   File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 201, in get     finish=finish)   File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 222, in multiget     column_count=column_count)   File "/usr/lib/python2.7/site-packages/cfgm_common/vnc_cassandra.py", line 514, in wrapper     return func(*args, **kwargs)   File "/usr/lib/python2.7/site-packages/pycassa/columnfamily.py", line 772, in multiget     packed_keys[offset:offset + buffer_size], cp, sp, consistency)   File "/usr/lib/python2.7/site-packages/pycassa/pool.py", line 577, in execute     return getattr(conn, f)(*args, **kwargs)   File "/usr/lib/python2.7/site-packages/pycassa/pool.py", line 127, in new_f     result = f(self, *args, **kwargs)   File "/usr/lib/python2.7/site-packages/pycassa/cassandra/Cassandra.py", line 669, in multiget_slice     return self.recv_multiget_slice()   File "/usr/lib/python2.7/site-packages/pycassa/cassandra/Cassandra.py", line 695, in recv_multiget_slice     raise result.ire InvalidRequestException: InvalidRequestException(why='unconfigured table obj_fq_name_table')
2018-06-19 03:56:52 Venkatesh Velpula juniperopenstack: milestone r5.1.1
2018-06-19 04:52:38 Sudheendra Rao juniperopenstack: milestone r5.1.1 r5.1.0
2018-06-19 04:53:03 Sudheendra Rao juniperopenstack: importance High Critical
2018-06-19 04:54:03 Sudheendra Rao juniperopenstack: assignee Sachchidanand Vaidya (vaidyasd)
2018-06-19 04:54:24 Sudheendra Rao nominated for series juniperopenstack/trunk
2018-06-19 04:54:24 Sudheendra Rao bug task added juniperopenstack/trunk
2018-06-19 05:07:02 Sachchidanand Vaidya juniperopenstack/trunk: assignee Sachchidanand Vaidya (vaidyasd) Dinesh Bakiaraj (dineshb)
2018-06-19 05:22:19 Venkatesh Velpula summary k8s_5.0:config api doesn't come after the restart k8s_5.0:config api doesn't come up after the restart
2018-06-19 08:21:05 Sachchidanand Vaidya juniperopenstack/trunk: assignee Dinesh Bakiaraj (dineshb) Shivayogi Ugaji (shivayogi123)
2018-06-19 20:51:56 Shivayogi Ugaji juniperopenstack/trunk: assignee Shivayogi Ugaji (shivayogi123) Nagendra Prasath (npchandran)
2018-07-04 03:04:22 Venkatesh Velpula tags contrail-control contrail-kube-manager sanityblocker blocker contrail-control contrail-kube-manager
2018-10-01 21:19:02 Ananth Suryanarayana tags blocker contrail-control contrail-kube-manager blocker contrail-kube-manager
2018-12-10 23:36:59 Nagendra Prasath juniperopenstack/trunk: importance Critical Medium