Observed this kube manager traceback while running sanity on k8s nested setup.
Traceback
----------
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_kubernetes.py", line 376, in vnc_process
self.pod_mgr.process(event)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 496, in process
pod_namespace, pod_node, labels, vm_vmi)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 373, in vnc_pod_update
pod_node, labels, vm_vmi)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 360, in vnc_pod_add
self._link_vm_to_node(vm_obj, pod_node)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 299, in _link_vm_to_node
%(self._name, vrouter_fq_name, pod_id))
NameError: global name 'pod_id' is not defined
When the crash is observed following traceback occurs on API server
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/vnc_cfg_api_server/vnc_cfg_api_server.py", line 3110, in _put_common
req_obj_dict, db_obj_dict)
File "/usr/lib/python2.7/dist-packages/stevedore/extension.py", line 252, in map_method
method_name, *args, **kwds)
File "/usr/lib/python2.7/dist-packages/stevedore/extension.py", line 225, in map
self._invoke_one_plugin(response.append, func, e, args, kwds)
File "/usr/lib/python2.7/dist-packages/stevedore/extension.py", line 256, in _invoke_one_plugin
response_callback(func(e, *args, **kwds))
File "/usr/lib/python2.7/dist-packages/stevedore/extension.py", line 230, in _call_extension_method
return getattr(extension.obj, method_name)(*args, **kwds)
File "/usr/lib/python2.7/dist-packages/vnc_openstack/__init__.py", line 882, in wrapper
return func(self, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/vnc_openstack/__init__.py", line 992, in post_virtual_network_update
ipam_refs = vn_dict.get('network_ipam_refs')
AttributeError: 'NoneType' object has no attribute 'get'
Details Traceback
-----------------
u'oper': u'CREATE',
u'request-id': u'req-bcb68a7c-32f9-4ada-b39d-b0e914a3347f',
u'type': u'floating_ip',
u'uuid': u'5e85a5d4-8146-11e7-bf6b-02a4b482907a'}
08/14/2017 11:14:49 PM [contrail-kube-manager]: __default__ [SYS_DEBUG]: KubeManagerDebugLog: Notification Message: {u'fq_name': [u'default-domain',
u'admin',
u'k8s-nested-cluster',
u'svc-fip-pool-k8s-nested-cluster',
u'cluster-svc-fip-ctest-busybox-pod-29334407__5e85a5d4-8146-11e7-bf6b-02a4b482907a'],
u'oper': u'UPDATE',
u'request-id': u'req-ebb89ee4-18bf-4b01-bed5-38a5f812d86d',
u'type': u'floating_ip',
u'uuid': u'5e85a5d4-8146-11e7-bf6b-02a4b482907a'}
08/14/2017 11:14:49 PM [contrail-kube-manager]: VncKubernetes - <type 'exceptions.NameError'>
Python 2.7.12: /usr/bin/python
Mon Aug 14 23:14:49 2017
A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.
/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_kubernetes.py in vnc_process(self=<kube_manager.vnc.vnc_kubernetes.VncKubernetes object>)
374 uid = metadata.get('uid')
375 if kind == 'Pod':
376 self.pod_mgr.process(event)
377 elif kind == 'Service':
378 self.service_mgr.process(event)
self = <kube_manager.vnc.vnc_kubernetes.VncKubernetes object>
self.pod_mgr = <kube_manager.vnc.vnc_pod.VncPod object>
self.pod_mgr.process = <bound method VncPod.process of <kube_manager.vnc.vnc_pod.VncPod object>>
event = {u'object': {u'apiVersion': u'v1', u'kind': u'Pod', u'metadata': {u'creationTimestamp': u'2017-08-14T23:14:48Z', u'name': u'ctest-busybox-pod-29334407', u'namespace': u'ctest-namespace-69012803', u'resourceVersion': u'165685', u'selfLink': u'/api/v1/namespaces/ctest-namespace-69012803/pods/ctest-busybox-pod-29334407', u'uid': u'5e5ce567-8146-11e7-b94d-02a4b482907a'}, u'spec': {u'containers': [{u'command': [u'sleep', u'1000000'], u'image': u'busybox', u'imagePullPolicy': u'IfNotPresent', u'name': u'ctest-busybox-pod-29334407-0', u'resources': {}, u'terminationMessagePath': u'/dev/termination-log', u'terminationMessagePolicy': u'File', u'volumeMounts': [{...}]}], u'dnsPolicy': u'ClusterFirst', u'nodeName': u'vm-test-3', u'restartPolicy': u'Always', u'schedulerName': u'default-scheduler', u'securityContext': {}, u'serviceAccount': u'default', u'serviceAccountName': u'default', u'terminationGracePeriodSeconds': 30, u'tolerations': [{u'effect': u'NoExecute', u'key': u'node.alpha.kubernetes.io/notReady', u'operator': u'Exists', u'tolerationSeconds': 300}, {u'effect': u'NoExecute', u'key': u'node.alpha.kubernetes.io/unreachable', u'operator': u'Exists', u'tolerationSeconds': 300}], ...}, u'status': {u'conditions': [{u'lastProbeTime': None, u'lastTransitionTime': u'2017-08-14T23:14:47Z', u'status': u'True', u'type': u'Initialized'}, {u'lastProbeTime': None, u'lastTransitionTime': u'2017-08-14T23:14:47Z', u'message': u'containers with unready status: [ctest-busybox-pod-29334407-0]', u'reason': u'ContainersNotReady', u'status': u'False', u'type': u'Ready'}, {u'lastProbeTime': None, u'lastTransitionTime': u'2017-08-14T23:14:48Z', u'status': u'True', u'type': u'PodScheduled'}], u'containerStatuses': [{u'image': u'busybox', u'imageID': u'', u'lastState': {}, u'name': u'ctest-busybox-pod-29334407-0', u'ready': False, u'restartCount': 0, u'state': {u'waiting': {...}}}], u'hostIP': u'1.1.1.5', u'phase': u'Pending', u'qosClass': u'BestEffort', u'startTime': u'2017-08-14T23:14:47Z'}}, u'type': u'MODIFIED'}
/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py in process(self=<kube_manager.vnc.vnc_pod.VncPod object>, event={u'object': {u'apiVersion': u'v1', u'kind': u'Pod', u'metadata': {u'creationTimestamp': u'2017-08-14T23:14:48Z', u'name': u'ctest-busybox-pod-29334407', u'namespace': u'ctest-namespace-69012803', u'resourceVersion': u'165685', u'selfLink': u'/api/v1/namespaces/ctest-namespace-69012803/pods/ctest-busybox-pod-29334407', u'uid': u'5e5ce567-8146-11e7-b94d-02a4b482907a'}, u'spec': {u'containers': [{u'command': [u'sleep', u'1000000'], u'image': u'busybox', u'imagePullPolicy': u'IfNotPresent', u'name': u'ctest-busybox-pod-29334407-0', u'resources': {}, u'terminationMessagePath': u'/dev/termination-log', u'terminationMessagePolicy': u'File', u'volumeMounts': [{...}]}], u'dnsPolicy': u'ClusterFirst', u'nodeName': u'vm-test-3', u'restartPolicy': u'Always', u'schedulerName': u'default-scheduler', u'securityContext': {}, u'serviceAccount': u'default', u'serviceAccountName': u'default', u'terminationGracePeriodSeconds': 30, u'tolerations': [{u'effect': u'NoExecute', u'key': u'node.alpha.kubernetes.io/notReady', u'operator': u'Exists', u'tolerationSeconds': 300}, {u'effect': u'NoExecute', u'key': u'node.alpha.kubernetes.io/unreachable', u'operator': u'Exists', u'tolerationSeconds': 300}], ...}, u'status': {u'conditions': [{u'lastProbeTime': None, u'lastTransitionTime': u'2017-08-14T23:14:47Z', u'status': u'True', u'type': u'Initialized'}, {u'lastProbeTime': None, u'lastTransitionTime': u'2017-08-14T23:14:47Z', u'message': u'containers with unready status: [ctest-busybox-pod-29334407-0]', u'reason': u'ContainersNotReady', u'status': u'False', u'type': u'Ready'}, {u'lastProbeTime': None, u'lastTransitionTime': u'2017-08-14T23:14:48Z', u'status': u'True', u'type': u'PodScheduled'}], u'containerStatuses': [{u'image': u'busybox', u'imageID': u'', u'lastState': {}, u'name': u'ctest-busybox-pod-29334407-0', u'ready': False, u'restartCount': 0, u'state': {u'waiting': {...}}}], u'hostIP': u'1.1.1.5', u'phase': u'Pen
494 else:
495 vm = self.vnc_pod_update(pod_id, pod_name,
496 pod_namespace, pod_node, labels, vm_vmi)
497 if vm:
498 self._network_policy_mgr.update_pod_np(pod_namespace, pod_id, labels)
pod_namespace = u'ctest-namespace-69012803'
pod_node = u'vm-test-3'
labels = {}
vm_vmi = <kube_manager.vnc.config_db.VirtualMachineInterfaceKM object>
/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py in vnc_pod_update(self=<kube_manager.vnc.vnc_pod.VncPod object>, pod_id=u'5e5ce567-8146-11e7-b94d-02a4b482907a', pod_name=u'ctest-busybox-pod-29334407', pod_namespace=u'ctest-namespace-69012803', pod_node=u'vm-test-3', labels={}, vm_vmi=<kube_manager.vnc.config_db.VirtualMachineInterfaceKM object>)
371 # If the vm is not created yet, do so now.
372 vm = self.vnc_pod_add(pod_id, pod_name, pod_namespace,
373 pod_node, labels, vm_vmi)
374 if not vm:
375 return
pod_node = u'vm-test-3'
labels = {}
vm_vmi = <kube_manager.vnc.config_db.VirtualMachineInterfaceKM object>
/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py in vnc_pod_add(self=<kube_manager.vnc.vnc_pod.VncPod object>, pod_id=u'5e5ce567-8146-11e7-b94d-02a4b482907a', pod_name=u'ctest-busybox-pod-29334407', pod_namespace=u'ctest-namespace-69012803', pod_node=u'vm-test-3', labels={}, vm_vmi=<kube_manager.vnc.config_db.VirtualMachineInterfaceKM object>)
358 self._create_cluster_service_fip(pod_name, pod_namespace, vmi_uuid)
359
360 self._link_vm_to_node(vm_obj, pod_node)
361 vm = VirtualMachineKM.locate(pod_id)
362 if vm:
self = <kube_manager.vnc.vnc_pod.VncPod object>
self._link_vm_to_node = <bound method VncPod._link_vm_to_node of <kube_manager.vnc.vnc_pod.VncPod object>>
vm_obj = <vnc_api.gen.resource_client.VirtualMachine object>
pod_node = u'vm-test-3'
/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py in _link_vm_to_node(self=<kube_manager.vnc.vnc_pod.VncPod object>, vm_obj=<vnc_api.gen.resource_client.VirtualMachine object>, pod_node=u'vm-test-3')
297 except Exception as e:
298 self._logger.debug("%s - Vrouter %s Not Found for Pod %s"
299 %(self._name, vrouter_fq_name, pod_id))
300 return
301
self = <kube_manager.vnc.vnc_pod.VncPod object>
self._name = 'VncPod'
vrouter_fq_name = ['default-global-system-config', u'vm-test-3']
pod_id undefined
<type 'exceptions.NameError'>: global name 'pod_id' is not defined
__class__ = <type 'exceptions.NameError'>
__delattr__ = <method-wrapper '__delattr__' of exceptions.NameError object>
__dict__ = {}
__doc__ = 'Name not found globally.'
__format__ = <built-in method __format__ of exceptions.NameError object>
__getattribute__ = <method-wrapper '__getattribute__' of exceptions.NameError object>
__getitem__ = <method-wrapper '__getitem__' of exceptions.NameError object>
__getslice__ = <method-wrapper '__getslice__' of exceptions.NameError object>
__hash__ = <method-wrapper '__hash__' of exceptions.NameError object>
__init__ = <method-wrapper '__init__' of exceptions.NameError object>
__new__ = <built-in method __new__ of type object>
__reduce__ = <built-in method __reduce__ of exceptions.NameError object>
__reduce_ex__ = <built-in method __reduce_ex__ of exceptions.NameError object>
__repr__ = <method-wrapper '__repr__' of exceptions.NameError object>
__setattr__ = <method-wrapper '__setattr__' of exceptions.NameError object>
__setstate__ = <built-in method __setstate__ of exceptions.NameError object>
__sizeof__ = <built-in method __sizeof__ of exceptions.NameError object>
__str__ = <method-wrapper '__str__' of exceptions.NameError object>
__subclasshook__ = <built-in method __subclasshook__ of type object>
__unicode__ = <built-in method __unicode__ of exceptions.NameError object>
args = ("global name 'pod_id' is not defined",)
message = "global name 'pod_id' is not defined"
The above is a description of an error in a Python program. Here is
the original traceback:
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_kubernetes.py", line 376, in vnc_process
self.pod_mgr.process(event)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 496, in process
pod_namespace, pod_node, labels, vm_vmi)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 373, in vnc_pod_update
pod_node, labels, vm_vmi)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 360, in vnc_pod_add
self._link_vm_to_node(vm_obj, pod_node)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 299, in _link_vm_to_node
%(self._name, vrouter_fq_name, pod_id))
NameError: global name 'pod_id' is not defined
Logs copied to /auto/cs-