After restart of the config api the pod creation fails .we hit this issue intermittently ,but if we hit this issue all the testcases will fail . right now i kept the setup intact in problem state ..could you please look at it. Build :5.1.0-250 Deployment :Ansible_deployer HOST OS: CENTOS7.5 ======================= Topology ================== vrouter +k8s_node: ip: nodec60 ip: nodec61 config +control++kubemanager: ip: nodeg12(k8s_master) ip: nodeg31 ip: nodec58 on agent ======== 018-09-19 Wed 13:08:05:766.554 IST nodec60 [Thread 140180966799104, Pid 1807]: [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:nodec60:vhost0 eps= [ [ _iter106->first = 00000000-0000-0000-0000-000000000001 [ workload = client= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 6291 out_bytes = 50500 in_pkts = 81 out_pkts = 94 action = pass ], ] ] server= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 50500 out_bytes = 6291 in_pkts = 94 out_pkts = 81 action = pass ], ] ] ], ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2018-09-19 Wed 13:08:07:439.498 IST nodec60 [Thread 140180958402304, Pid 1807]: SANDESH: Sending: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] : 2053 2018-09-19 Wed 13:08:07:439.910 IST nodec60 [Thread 140180958402304, Pid 1807]: SANDESH: Sending: LEVEL: [ SYS_DEBUG ] -> [ INVALID ] : 0 2018-09-19 Wed 13:08:35:767.377 IST nodec60 [Thread 140180970997504, Pid 1807]: [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:nodec60:vhost0 eps= [ [ _iter106->first = 00000000-0000-0000-0000-000000000001 [ workload = client= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 6446 out_bytes = 46142 in_pkts = 84 out_pkts = 86 action = pass ], ] ] server= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 46142 out_bytes = 6446 in_pkts = 86 out_pkts = 84 action = pass ], ] ] ], ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135 2018-09-19 Wed 13:08:37:440.518 IST nodec60 [Thread 140180954203904, Pid 1807]: SANDESH: Sending: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] : 2185 2018-09-19 Wed 13:08:37:440.917 IST nodec60 [Thread 140180954203904, Pid 1807]: SANDESH: Sending: LEVEL: [ SYS_DEBUG ] -> [ INVALID ] : 0 cni logs ========= E : 27646 : 2018/09/19 13:10:30 contrail-kube-cni.go:68: Failed processing Add command. I : 27880 : 2018/09/19 13:10:32 contrail-kube-cni.go:53: Came in Add for container 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5 I : 27880 : 2018/09/19 13:10:32 contrail-kube-cni.go:41: getPodInfo success. container-id 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5 uuid 1ad2a993-bbdf-11e8-88fd-002590c476a0 name test-75c49697d7-lq7bs I : 27880 : 2018/09/19 13:10:32 cni.go:88: ContainerID : 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5 I : 27880 : 2018/09/19 13:10:32 cni.go:89: NetNS : /proc/27836/ns/net I : 27880 : 2018/09/19 13:10:32 cni.go:90: Container Ifname : eth0 I : 27880 : 2018/09/19 13:10:32 cni.go:91: Args : IgnoreUnknown=1;K8S_POD_NAMESPACE=default;K8S_POD_NAME=test-75c49697d7-lq7bs;K8S_POD_INFRA_CONTAINER_ID=86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5 I : 27880 : 2018/09/19 13:10:32 cni.go:92: CNI VERSION : 0.2.0 I : 27880 : 2018/09/19 13:10:32 cni.go:93: MTU : 1500 I : 27880 : 2018/09/19 13:10:32 cni.go:94: Config File : {"cniVersion":"0.2.0","contrail":{"config-dir":"/var/lib/contrail/ports/vm","log-file":"/var/log/contrail/cni/opencontrail.log","log-level":"4","poll-retries":15,"poll-timeout":5,"vrouter-ip":"127.0.0.1","vrouter-port":9091},"name":"contrail-k8s-cni","type":"contrail-k8s-cni"} I : 27880 : 2018/09/19 13:10:32 cni.go:95: &{cniArgs:0xc4202ab340 Mode:k8s VifType:veth VifParent:eth0 LogDir:/var/log/contrail/cni LogFile:/var/log/contrail/cni/opencontrail.log LogLevel:4 Mtu:1500 ContainerUuid:1ad2a993-bbdf-11e8-88fd-002590c476a0 ContainerName:test-75c49697d7-lq7bs ContainerVn: VRouter:{Server:127.0.0.1 Port:9091 Dir:/var/lib/contrail/ports/vm PollTimeout:5 PollRetries:15 containerId: containerUuid: containerVn: httpClient:0xc4201a3710}} I : 27880 : 2018/09/19 13:10:32 vrouter.go:446: {Server:127.0.0.1 Port:9091 Dir:/var/lib/contrail/ports/vm PollTimeout:5 PollRetries:15 containerId: containerUuid: containerVn: httpClient:0xc4201a3710} I : 27880 : 2018/09/19 13:10:32 vrouter.go:79: VRouter request. Operation : GET Url : http://127.0.0.1:9091/vm-cfg/1ad2a993-bbdf-11e8-88fd-002590c476a0 E : 27880 : 2018/09/19 13:10:32 vrouter.go:147: Failed HTTP Get operation. Return code 404 I : 27880 : 2018/09/19 13:10:32 vrouter.go:181: Iteration 0 : Get vrouter failed [root@nodec60 contrail]# contrail-status Pod Service Original Name State Status vrouter agent contrail-vrouter-agent running Up 19 hours vrouter nodemgr contrail-nodemgr running Up 19 hours vrouter kernel module is PRESENT == Contrail vrouter == nodemgr: active agent: active [root@nodec60 contrail] config api =========== u'request-id': u'req-d432efc2-1912-42dc-93e6-fd78fe859311', u'type': u'virtual_machine_interface', u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'} 09/19/2018 01:09:13 PM [contrail-api] [DEBUG]: Add uve in the [ObjectVMITable:ContrailConfigTrace] map 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 6051 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >> 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiConfigLog: api_log = << identifier_uuid = 1af781aa-bbdf-11e8-b7ef-002590c55f6a object_type = virtual_machine_interface identifier_name = default-domain:k8s-default:test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a url = http://127.0.0.1/ref-update operation = ref-update domain = default-domain >> 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1422 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >> 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1489 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >> 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 899 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >> 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = SEND application = CASSANDRA response_time_in_usec = 1413 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >> 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 877 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >> 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiStatsLog: api_stats = << operation_type = POST user = useragent = nodeg31:/usr/bin/contrail-kube-manager remote_ip = 10.204.217.71 domain_name = default-domain project_name = default-project object_type = virtual_machine_interface response_time_in_usec = 42966 response_size = 529 resp_code = 200 >> 09/19/2018 01:09:13 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'default-domain', u'k8s-default', u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'], u'oper': u'UPDATE', u'request-id': u'req-d432efc2-1912-42dc-93e6-fd78fe859311', u'type': u'virtual_machine_interface', u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'} 09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1117 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >> 09/19/2018 01:09:14 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'default-domain', u'k8s-default', u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'], u'oper': u'UPDATE', u'request-id': u'req-c2ac9e16-9f35-4e4f-92c5-4f26372e9022', u'type': u'virtual_machine_interface', u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'} 09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1219 response_size = 0 identifier = req-c2ac9e16-9f35-4e4f-92c5-4f26372e9022 >> 09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1144 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >> 09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiStatsLog: api_stats = << operation_type = GET user = useragent = nodeg31:/usr/bin/contrail-kube-manager remote_ip = 10.204.217.71 domain_name = default-domain project_name = default-project object_type = virtual_machine_interface response_time_in_usec = 2405 response_size = 2666 resp_code = 200 >> 09/19/2018 01:09:14 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a'], u'obj_dict': {u'display_name': u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a', u'fq_name': [u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a'], u'id_perms': {u'created': u'2018-09-19T07:39:14.035968', u'creator': None, u'description': None, u'enable': True, u'last_modified': u'2018-09-19T07:39:14.035968', u'permissions': {u'group': u'cloud-admin-group', u'group_access': 7, u'other_access': 7, u'owner': u'cloud-admin', u'owner_access': 7}, u'user_visible': True, u'uuid': {u'uuid_lslong': 13253812389717303146L, u'uuid_mslong': 1946120732318568936}}, u'instance_ip_address': u'10.47.255.251', u'perms2': {u'global_access': 0, u'owner': u'cloud-admin', u'owner_access': 7, u'share': []}, u'subnet_uuid': u'eff1f49d-cd1b-459b-a5d8-31a54440f83f', u'uuid': u'1b02027e-bbdf-11e8-b7ef-002590c55f6a', u'virtual_machine_interface_refs': [{u'to': [u'default-domain', u'k8s-default', u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'], u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}], u'virtual_network_refs': [{u'to': [u'default-domain', u'k8s-default', u'k8s-default-pod-network'], u'uuid': u'c2ac9d50-27bb-4d18-b6c3-715bc88506a0'}]}, == Contrail control == control: active nodemgr: active named: active dns: active == Contrail config-database == nodemgr: initializing (Disk for DB is too low. ) zookeeper: active rabbitmq: active cassandra: active == Contrail kubernetes == kube-manager: active == Contrail database == kafka: active nodemgr: initializing (Disk for DB is too low. ) zookeeper: active cassandra: active == Contrail analytics == snmp-collector: active query-engine: active api: active alarm-gen: active nodemgr: active collector: active topology: active == Contrail webui == web: active job: active == Contrail config == svc-monitor: active nodemgr: active device-manager: active api: active schema: active