| 2018-09-19 08:14:03 |
Venkatesh Velpula |
description |
After restart of the config api the pod creation fails .we hit this issue intermittently ,but if we hit this issue all the testcases will fail .
right now i kept the setup intact in problem state ..could you please look at it.
Build :5.1.0-250
Deployment :Ansible_deployer
HOST OS: CENTOS7.5
=======================
Topology
==================
vrouter +k8s_node:
ip: nodec60
ip: nodec61
config +control++kubemanager:
ip: nodeg12(k8s_master)
ip: nodeg31
ip: nodec58
on agent
========
018-09-19 Wed 13:08:05:766.554 IST nodec60 [Thread 140180966799104, Pid 1807]: [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:nodec60:vhost0 eps= [ [ _iter106->first = 00000000-0000-0000-0000-000000000001 [ workload = client= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 6291 out_bytes = 50500 in_pkts = 81 out_pkts = 94 action = pass ], ] ] server= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 50500 out_bytes = 6291 in_pkts = 94 out_pkts = 81 action = pass ], ] ] ], ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135
2018-09-19 Wed 13:08:07:439.498 IST nodec60 [Thread 140180958402304, Pid 1807]: SANDESH: Sending: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] : 2053
2018-09-19 Wed 13:08:07:439.910 IST nodec60 [Thread 140180958402304, Pid 1807]: SANDESH: Sending: LEVEL: [ SYS_DEBUG ] -> [ INVALID ] : 0
2018-09-19 Wed 13:08:35:767.377 IST nodec60 [Thread 140180970997504, Pid 1807]: [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:nodec60:vhost0 eps= [ [ _iter106->first = 00000000-0000-0000-0000-000000000001 [ workload = client= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 6446 out_bytes = 46142 in_pkts = 84 out_pkts = 86 action = pass ], ] ] server= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 46142 out_bytes = 6446 in_pkts = 86 out_pkts = 84 action = pass ], ] ] ], ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135
2018-09-19 Wed 13:08:37:440.518 IST nodec60 [Thread 140180954203904, Pid 1807]: SANDESH: Sending: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] : 2185
2018-09-19 Wed 13:08:37:440.917 IST nodec60 [Thread 140180954203904, Pid 1807]: SANDESH: Sending: LEVEL: [ SYS_DEBUG ] -> [ INVALID ] : 0
cni logs
=========
E : 27646 : 2018/09/19 13:10:30 contrail-kube-cni.go:68: Failed processing Add command.
I : 27880 : 2018/09/19 13:10:32 contrail-kube-cni.go:53: Came in Add for container 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5
I : 27880 : 2018/09/19 13:10:32 contrail-kube-cni.go:41: getPodInfo success. container-id 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5 uuid 1ad2a993-bbdf-11e8-88fd-002590c476a0 name test-75c49697d7-lq7bs
I : 27880 : 2018/09/19 13:10:32 cni.go:88: ContainerID : 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5
I : 27880 : 2018/09/19 13:10:32 cni.go:89: NetNS : /proc/27836/ns/net
I : 27880 : 2018/09/19 13:10:32 cni.go:90: Container Ifname : eth0
I : 27880 : 2018/09/19 13:10:32 cni.go:91: Args : IgnoreUnknown=1;K8S_POD_NAMESPACE=default;K8S_POD_NAME=test-75c49697d7-lq7bs;K8S_POD_INFRA_CONTAINER_ID=86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5
I : 27880 : 2018/09/19 13:10:32 cni.go:92: CNI VERSION : 0.2.0
I : 27880 : 2018/09/19 13:10:32 cni.go:93: MTU : 1500
I : 27880 : 2018/09/19 13:10:32 cni.go:94: Config File : {"cniVersion":"0.2.0","contrail":{"config-dir":"/var/lib/contrail/ports/vm","log-file":"/var/log/contrail/cni/opencontrail.log","log-level":"4","poll-retries":15,"poll-timeout":5,"vrouter-ip":"127.0.0.1","vrouter-port":9091},"name":"contrail-k8s-cni","type":"contrail-k8s-cni"}
I : 27880 : 2018/09/19 13:10:32 cni.go:95: &{cniArgs:0xc4202ab340 Mode:k8s VifType:veth VifParent:eth0 LogDir:/var/log/contrail/cni LogFile:/var/log/contrail/cni/opencontrail.log LogLevel:4 Mtu:1500 ContainerUuid:1ad2a993-bbdf-11e8-88fd-002590c476a0 ContainerName:test-75c49697d7-lq7bs ContainerVn: VRouter:{Server:127.0.0.1 Port:9091 Dir:/var/lib/contrail/ports/vm PollTimeout:5 PollRetries:15 containerId: containerUuid: containerVn: httpClient:0xc4201a3710}}
I : 27880 : 2018/09/19 13:10:32 vrouter.go:446: {Server:127.0.0.1 Port:9091 Dir:/var/lib/contrail/ports/vm PollTimeout:5 PollRetries:15 containerId: containerUuid: containerVn: httpClient:0xc4201a3710}
I : 27880 : 2018/09/19 13:10:32 vrouter.go:79: VRouter request. Operation : GET Url : http://127.0.0.1:9091/vm-cfg/1ad2a993-bbdf-11e8-88fd-002590c476a0
E : 27880 : 2018/09/19 13:10:32 vrouter.go:147: Failed HTTP Get operation. Return code 404
I : 27880 : 2018/09/19 13:10:32 vrouter.go:181: Iteration 0 : Get vrouter failed
[root@nodec60 contrail]# contrail-status
Pod Service Original Name State Status
vrouter agent contrail-vrouter-agent running Up 19 hours
vrouter nodemgr contrail-nodemgr running Up 19 hours
vrouter kernel module is PRESENT
== Contrail vrouter ==
nodemgr: active
agent: active
[root@nodec60 contrail]
config api
===========
u'request-id': u'req-d432efc2-1912-42dc-93e6-fd78fe859311',
u'type': u'virtual_machine_interface',
u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}
09/19/2018 01:09:13 PM [contrail-api] [DEBUG]: Add uve <default-domain:k8s-default:test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a, 1873> in the [ObjectVMITable:ContrailConfigTrace] map
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 6051 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiConfigLog: api_log = << identifier_uuid = 1af781aa-bbdf-11e8-b7ef-002590c55f6a object_type = virtual_machine_interface identifier_name = default-domain:k8s-default:test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a url = http://127.0.0.1/ref-update operation = ref-update domain = default-domain >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1422 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1489 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 899 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = SEND application = CASSANDRA response_time_in_usec = 1413 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 877 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiStatsLog: api_stats = << operation_type = POST user = useragent = nodeg31:/usr/bin/contrail-kube-manager remote_ip = 10.204.217.71 domain_name = default-domain project_name = default-project object_type = virtual_machine_interface response_time_in_usec = 42966 response_size = 529 resp_code = 200 >>
09/19/2018 01:09:13 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'default-domain',
u'k8s-default',
u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'],
u'oper': u'UPDATE',
u'request-id': u'req-d432efc2-1912-42dc-93e6-fd78fe859311',
u'type': u'virtual_machine_interface',
u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1117 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:14 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'default-domain',
u'k8s-default',
u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'],
u'oper': u'UPDATE',
u'request-id': u'req-c2ac9e16-9f35-4e4f-92c5-4f26372e9022',
u'type': u'virtual_machine_interface',
u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}
09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1219 response_size = 0 identifier = req-c2ac9e16-9f35-4e4f-92c5-4f26372e9022 >>
09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1144 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiStatsLog: api_stats = << operation_type = GET user = useragent = nodeg31:/usr/bin/contrail-kube-manager remote_ip = 10.204.217.71 domain_name = default-domain project_name = default-project object_type = virtual_machine_interface response_time_in_usec = 2405 response_size = 2666 resp_code = 200 >>
09/19/2018 01:09:14 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a'],
u'obj_dict': {u'display_name': u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a',
u'fq_name': [u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a'],
u'id_perms': {u'created': u'2018-09-19T07:39:14.035968',
u'creator': None,
u'description': None,
u'enable': True,
u'last_modified': u'2018-09-19T07:39:14.035968',
u'permissions': {u'group': u'cloud-admin-group',
u'group_access': 7,
u'other_access': 7,
u'owner': u'cloud-admin',
u'owner_access': 7},
u'user_visible': True,
u'uuid': {u'uuid_lslong': 13253812389717303146L,
u'uuid_mslong': 1946120732318568936}},
u'instance_ip_address': u'10.47.255.251',
u'perms2': {u'global_access': 0,
u'owner': u'cloud-admin',
u'owner_access': 7,
u'share': []},
u'subnet_uuid': u'eff1f49d-cd1b-459b-a5d8-31a54440f83f',
u'uuid': u'1b02027e-bbdf-11e8-b7ef-002590c55f6a',
u'virtual_machine_interface_refs': [{u'to': [u'default-domain',
u'k8s-default',
u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'],
u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}],
u'virtual_network_refs': [{u'to': [u'default-domain',
u'k8s-default',
u'k8s-default-pod-network'],
u'uuid': u'c2ac9d50-27bb-4d18-b6c3-715bc88506a0'}]},
== Contrail control ==
control: active
nodemgr: active
named: active
dns: active
== Contrail config-database ==
nodemgr: initializing (Disk for DB is too low. )
zookeeper: active
rabbitmq: active
cassandra: active
== Contrail kubernetes ==
kube-manager: active
== Contrail database ==
kafka: active
nodemgr: initializing (Disk for DB is too low. )
zookeeper: active
cassandra: active
== Contrail analytics ==
snmp-collector: active
query-engine: active
api: active
alarm-gen: active
nodemgr: active
collector: active
topology: active
== Contrail webui ==
web: active
job: active
== Contrail config ==
svc-monitor: active
nodemgr: active
device-manager: active
api: active
schema: active |
After restart of the config api the pod creation fails .we hit this issue intermittently ,but if we hit this issue all the testcases will fail after teh config api restart .
right now i kept the setup intact in problem state ..could you please look at it.
Build :5.1.0-250
Deployment :Ansible_deployer
HOST OS: CENTOS7.5
=======================
Topology
==================
vrouter +k8s_node:
ip: nodec60
ip: nodec61
config +control++kubemanager:
ip: nodeg12(k8s_master)
ip: nodeg31
ip: nodec58
on agent
========
018-09-19 Wed 13:08:05:766.554 IST nodec60 [Thread 140180966799104, Pid 1807]: [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:nodec60:vhost0 eps= [ [ _iter106->first = 00000000-0000-0000-0000-000000000001 [ workload = client= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 6291 out_bytes = 50500 in_pkts = 81 out_pkts = 94 action = pass ], ] ] server= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 50500 out_bytes = 6291 in_pkts = 94 out_pkts = 81 action = pass ], ] ] ], ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135
2018-09-19 Wed 13:08:07:439.498 IST nodec60 [Thread 140180958402304, Pid 1807]: SANDESH: Sending: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] : 2053
2018-09-19 Wed 13:08:07:439.910 IST nodec60 [Thread 140180958402304, Pid 1807]: SANDESH: Sending: LEVEL: [ SYS_DEBUG ] -> [ INVALID ] : 0
2018-09-19 Wed 13:08:35:767.377 IST nodec60 [Thread 140180970997504, Pid 1807]: [SYS_INFO]: EndpointSecurityStats: name = default-global-system-config:nodec60:vhost0 eps= [ [ _iter106->first = 00000000-0000-0000-0000-000000000001 [ workload = client= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 6446 out_bytes = 46142 in_pkts = 84 out_pkts = 86 action = pass ], ] ] server= [ [ [ app = tier = site = deployment = labels = custom_tags = remote_app_id = remote_tier_id = remote_site_id = remote_deployment_id = remote_label_ids = remote_custom_tag_ids = remote_prefix = remote_vn = default-domain:default-project:ip-fabric local_vn = default-domain:default-project:ip-fabric added = 0 deleted = 0 active = 7 in_bytes = 46142 out_bytes = 6446 in_pkts = 86 out_pkts = 84 action = pass ], ] ] ], ] ] file = controller/src/vnsw/agent/uve/interface_uve_stats_table.cc line = 135
2018-09-19 Wed 13:08:37:440.518 IST nodec60 [Thread 140180954203904, Pid 1807]: SANDESH: Sending: LEVEL: [ INVALID ] -> [ SYS_DEBUG ] : 2185
2018-09-19 Wed 13:08:37:440.917 IST nodec60 [Thread 140180954203904, Pid 1807]: SANDESH: Sending: LEVEL: [ SYS_DEBUG ] -> [ INVALID ] : 0
cni logs
=========
E : 27646 : 2018/09/19 13:10:30 contrail-kube-cni.go:68: Failed processing Add command.
I : 27880 : 2018/09/19 13:10:32 contrail-kube-cni.go:53: Came in Add for container 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5
I : 27880 : 2018/09/19 13:10:32 contrail-kube-cni.go:41: getPodInfo success. container-id 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5 uuid 1ad2a993-bbdf-11e8-88fd-002590c476a0 name test-75c49697d7-lq7bs
I : 27880 : 2018/09/19 13:10:32 cni.go:88: ContainerID : 86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5
I : 27880 : 2018/09/19 13:10:32 cni.go:89: NetNS : /proc/27836/ns/net
I : 27880 : 2018/09/19 13:10:32 cni.go:90: Container Ifname : eth0
I : 27880 : 2018/09/19 13:10:32 cni.go:91: Args : IgnoreUnknown=1;K8S_POD_NAMESPACE=default;K8S_POD_NAME=test-75c49697d7-lq7bs;K8S_POD_INFRA_CONTAINER_ID=86d6db19312882197c71999e7fb13f51dd6866ab0310061655ff63e91f90acd5
I : 27880 : 2018/09/19 13:10:32 cni.go:92: CNI VERSION : 0.2.0
I : 27880 : 2018/09/19 13:10:32 cni.go:93: MTU : 1500
I : 27880 : 2018/09/19 13:10:32 cni.go:94: Config File : {"cniVersion":"0.2.0","contrail":{"config-dir":"/var/lib/contrail/ports/vm","log-file":"/var/log/contrail/cni/opencontrail.log","log-level":"4","poll-retries":15,"poll-timeout":5,"vrouter-ip":"127.0.0.1","vrouter-port":9091},"name":"contrail-k8s-cni","type":"contrail-k8s-cni"}
I : 27880 : 2018/09/19 13:10:32 cni.go:95: &{cniArgs:0xc4202ab340 Mode:k8s VifType:veth VifParent:eth0 LogDir:/var/log/contrail/cni LogFile:/var/log/contrail/cni/opencontrail.log LogLevel:4 Mtu:1500 ContainerUuid:1ad2a993-bbdf-11e8-88fd-002590c476a0 ContainerName:test-75c49697d7-lq7bs ContainerVn: VRouter:{Server:127.0.0.1 Port:9091 Dir:/var/lib/contrail/ports/vm PollTimeout:5 PollRetries:15 containerId: containerUuid: containerVn: httpClient:0xc4201a3710}}
I : 27880 : 2018/09/19 13:10:32 vrouter.go:446: {Server:127.0.0.1 Port:9091 Dir:/var/lib/contrail/ports/vm PollTimeout:5 PollRetries:15 containerId: containerUuid: containerVn: httpClient:0xc4201a3710}
I : 27880 : 2018/09/19 13:10:32 vrouter.go:79: VRouter request. Operation : GET Url : http://127.0.0.1:9091/vm-cfg/1ad2a993-bbdf-11e8-88fd-002590c476a0
E : 27880 : 2018/09/19 13:10:32 vrouter.go:147: Failed HTTP Get operation. Return code 404
I : 27880 : 2018/09/19 13:10:32 vrouter.go:181: Iteration 0 : Get vrouter failed
[root@nodec60 contrail]# contrail-status
Pod Service Original Name State Status
vrouter agent contrail-vrouter-agent running Up 19 hours
vrouter nodemgr contrail-nodemgr running Up 19 hours
vrouter kernel module is PRESENT
== Contrail vrouter ==
nodemgr: active
agent: active
[root@nodec60 contrail]
config api
===========
u'request-id': u'req-d432efc2-1912-42dc-93e6-fd78fe859311',
u'type': u'virtual_machine_interface',
u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}
09/19/2018 01:09:13 PM [contrail-api] [DEBUG]: Add uve <default-domain:k8s-default:test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a, 1873> in the [ObjectVMITable:ContrailConfigTrace] map
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 6051 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiConfigLog: api_log = << identifier_uuid = 1af781aa-bbdf-11e8-b7ef-002590c55f6a object_type = virtual_machine_interface identifier_name = default-domain:k8s-default:test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a url = http://127.0.0.1/ref-update operation = ref-update domain = default-domain >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1422 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1489 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 899 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = SEND application = CASSANDRA response_time_in_usec = 1413 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 877 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiStatsLog: api_stats = << operation_type = POST user = useragent = nodeg31:/usr/bin/contrail-kube-manager remote_ip = 10.204.217.71 domain_name = default-domain project_name = default-project object_type = virtual_machine_interface response_time_in_usec = 42966 response_size = 529 resp_code = 200 >>
09/19/2018 01:09:13 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'default-domain',
u'k8s-default',
u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'],
u'oper': u'UPDATE',
u'request-id': u'req-d432efc2-1912-42dc-93e6-fd78fe859311',
u'type': u'virtual_machine_interface',
u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}
09/19/2018 01:09:13 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1117 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:14 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'default-domain',
u'k8s-default',
u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'],
u'oper': u'UPDATE',
u'request-id': u'req-c2ac9e16-9f35-4e4f-92c5-4f26372e9022',
u'type': u'virtual_machine_interface',
u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}
09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1219 response_size = 0 identifier = req-c2ac9e16-9f35-4e4f-92c5-4f26372e9022 >>
09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiLatencyStatsLog: node_name = issu-vm6 api_latency_stats = << operation_type = MULTIGET application = CASSANDRA response_time_in_usec = 1144 response_size = 0 identifier = req-d432efc2-1912-42dc-93e6-fd78fe859311 >>
09/19/2018 01:09:14 PM [contrail-api] [INFO]: __default__ [SYS_INFO]: VncApiStatsLog: api_stats = << operation_type = GET user = useragent = nodeg31:/usr/bin/contrail-kube-manager remote_ip = 10.204.217.71 domain_name = default-domain project_name = default-project object_type = virtual_machine_interface response_time_in_usec = 2405 response_size = 2666 resp_code = 200 >>
09/19/2018 01:09:14 PM [contrail-api] [DEBUG]: __default__ [SYS_DEBUG]: VncApiDebug: Notification Message: {u'fq_name': [u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a'],
u'obj_dict': {u'display_name': u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a',
u'fq_name': [u'test-75c49697d7-lq7bs__1b02027e-bbdf-11e8-b7ef-002590c55f6a'],
u'id_perms': {u'created': u'2018-09-19T07:39:14.035968',
u'creator': None,
u'description': None,
u'enable': True,
u'last_modified': u'2018-09-19T07:39:14.035968',
u'permissions': {u'group': u'cloud-admin-group',
u'group_access': 7,
u'other_access': 7,
u'owner': u'cloud-admin',
u'owner_access': 7},
u'user_visible': True,
u'uuid': {u'uuid_lslong': 13253812389717303146L,
u'uuid_mslong': 1946120732318568936}},
u'instance_ip_address': u'10.47.255.251',
u'perms2': {u'global_access': 0,
u'owner': u'cloud-admin',
u'owner_access': 7,
u'share': []},
u'subnet_uuid': u'eff1f49d-cd1b-459b-a5d8-31a54440f83f',
u'uuid': u'1b02027e-bbdf-11e8-b7ef-002590c55f6a',
u'virtual_machine_interface_refs': [{u'to': [u'default-domain',
u'k8s-default',
u'test-75c49697d7-lq7bs__1af781aa-bbdf-11e8-b7ef-002590c55f6a'],
u'uuid': u'1af781aa-bbdf-11e8-b7ef-002590c55f6a'}],
u'virtual_network_refs': [{u'to': [u'default-domain',
u'k8s-default',
u'k8s-default-pod-network'],
u'uuid': u'c2ac9d50-27bb-4d18-b6c3-715bc88506a0'}]},
== Contrail control ==
control: active
nodemgr: active
named: active
dns: active
== Contrail config-database ==
nodemgr: initializing (Disk for DB is too low. )
zookeeper: active
rabbitmq: active
cassandra: active
== Contrail kubernetes ==
kube-manager: active
== Contrail database ==
kafka: active
nodemgr: initializing (Disk for DB is too low. )
zookeeper: active
cassandra: active
== Contrail analytics ==
snmp-collector: active
query-engine: active
api: active
alarm-gen: active
nodemgr: active
collector: active
topology: active
== Contrail webui ==
web: active
job: active
== Contrail config ==
svc-monitor: active
nodemgr: active
device-manager: active
api: active
schema: active |
|
