R4.0 Build 4 Ubuntu 16.04.2 Container setup
After applying workaround for bug 1690488, kube-manager got vm config, but
kube-dns pod is not launched
[root@testbed-1-vm1 ~]# kubectl get pods --namespace=kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE
etcd-testbed-1-vm1 1/1 Running 0 1h 10.204.217.194 testbed-1-vm1
kube-apiserver-testbed-1-vm1 1/1 Running 0 1h 10.204.217.194 testbed-1-vm1
kube-controller-manager-testbed-1-vm1 1/1 Running 1 1h 10.204.217.194 testbed-1-vm1
kube-dns-3121064917-4t42w 0/3 ContainerCreating 0 1h <none> testbed-1-vm3
kube-dns-3913472980-n6tfd 3/3 Running 0 1h 172.17.0.2 testbed-1-vm1
kube-proxy-0558k 1/1 Running 0 1h 10.204.217.194 testbed-1-vm1
kube-proxy-hpg65 1/1 Running 0 1h 10.204.217.197 testbed-1-vm2
kube-proxy-krxfc 1/1 Running 0 1h 10.204.217.198 testbed-1-vm3
kube-scheduler-testbed-1-vm1 1/1 Running 1 1h 10.204.217.194 testbed-1-vm1
[root@testbed-1-vm1 ~]#
Log :
05/16/2017 06:35:53 AM [contrail-kube-manager]: __default__ [SYS_ERR]: KubeManagerErrorLog: VncKubernetes - <type 'exceptions.TypeError'>
Python 2.7.12: /usr/bin/python
Tue May 16 06:35:53 2017
A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.
/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_kubernetes.py in vnc_process(self=<kube_manager.vnc.vnc_kubernetes.VncKubernetes object>)
351 uid = metadata.get('uid')
352 if kind == 'Pod':
353 self.pod_mgr.process(event)
354 elif kind == 'Service':
355 self.service_mgr.process(event)
self = <kube_manager.vnc.vnc_kubernetes.VncKubernetes object>
self.pod_mgr = <kube_manager.vnc.vnc_pod.VncPod object>
self.pod_mgr.process = <bound method VncPod.process of <kube_manager.vnc.vnc_pod.VncPod object>>
self.pod_mgr.process = <bound method VncPod.process of <kube_manager.vnc.vnc_pod.VncPod object>>
event = {u'object': {u'apiVersion': u'v1', u'kind': u'Pod', u'metadata': {u'annotations': {u'kubernetes.io/created-by': u'{"kind":"SerializedReference","apiVersion":"v1...rsion":"extensions","resourceVersion":"1057"}}\n', u'scheduler.alpha.kubernetes.io/critical-pod': u''}, u'creationTimestamp': u'2017-05-16T05:15:46Z', u'generateName': u'kube-dns-3121064917-', u'labels': {u'k8s-app': u'kube-dns', 'namespace': None, u'pod-template-hash': u'3121064917'}, u'name': u'kube-dns-3121064917-4t42w', u'namespace': u'kube-system', u'ownerReferences': [{u'apiVersion': u'extensions/v1beta1', u'blockOwnerDeletion': True, u'controller': True, u'kind': u'ReplicaSet', u'name': u'kube-dns-3121064917', u'uid': u'b6dab065-39f6-11e7-81a4-525400010001'}], u'resourceVersion': u'1089', u'selfLink': u'/api/v1/namespaces/kube-system/pods/kube-dns-3121064917-4t42w', u'uid': u'b78011a8-39f6-11e7-81a4-525400010001'}, u'spec': {u'containers': [{u'args': [u'--domain=cluster.local.', u'--dns-port=10053', u'--config-dir=/kube-dns-config', u'--v=2'], u'env': [{...}], u'image': u'gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1', u'imagePullPolicy': u'IfNotPresent', u'livenessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'name': u'kubedns', u'ports': [{...}, {...}, {...}], u'readinessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'resources': {u'limits': {...}, u'requests': {...}}, u'terminationMessagePath': u'/dev/termination-log', ...}, {u'args': [u'-v=2', u'-logtostderr', u'-configDir=/etc/k8s/dns/dnsmasq-nanny', u'-restartDnsmasq=true', u'--', u'-k', u'--cache-size=1000', u'--log-facility=-', u'--server=/cluster.local/127.0.0.1#10053', u'--server=/in-addr.arpa/127.0.0.1#10053', u'--server=/ip6.arpa/127.0.0.1#10053'], u'image': u'gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.1', u'imagePullPolicy': u'IfNotPresent', u'livenessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'name': u'dnsmasq', u'ports': [{...}, {...}], u'readinessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'resources': {u'requests': {...}}, u'terminationMessagePath': u'/dev/termination-log', u'terminationMessagePolicy': u'File', ...}, {u'args': [u'--v=2', u'--logtostderr', u'--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A', u'--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A'], u'image': u'gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.1', u'imagePullPolicy': u'IfNotPresent', u'livenessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'name': u'sidecar', u'ports': [{...}], u'readinessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'resources': {u'requests': {...}}, u'terminationMessagePath': u'/dev/termination-log', u'terminationMessagePolicy': u'File', ...}], u'dnsPolicy': u'Default', u'nodeName': u'testbed-1-vm3', u'restartPolicy': u'Always', u'schedulerName': u'default-scheduler', u'securityContext': {}, u'serviceAccount': u'kube-dns', u'serviceAccountName': u'kube-dns', u'terminationGracePeriodSeconds': 30, u'tolerations': [{u'effect': u'NoExecute', u'key': u'node.alpha.kubernetes.io/notReady', u'operator': u'Exists', u'tolerationSeconds': 300}, {u'effect': u'NoExecute', u'key': u'node.alpha.kubernetes.io/unreachable', u'operator': u'Exists', u'tolerationSeconds': 300}], ...}, u'status': {u'conditions': [{u'lastProbeTime': None, u'lastTransitionTime': u'2017-05-16T05:15:51Z', u'status': u'True', u'type': u'Initialized'}, {u'lastProbeTime': None, u'lastTransitionTime': u'2017-05-16T05:15:51Z', u'message': u'containers with unready status: [kubedns dnsmasq sidecar]', u'reason': u'ContainersNotReady', u'status': u'False', u'type': u'Ready'}, {u'lastProbeTime': None, u'lastTransitionTime': u'2017-05-16T05:15:46Z', u'status': u'True', u'type': u'PodScheduled'}], u'containerStatuses': [{u'image': u'gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.1', u'imageID': u'', u'lastState': {}, u'name': u'dnsmasq', u'ready': False, u'restartCount': 0, u'state': {u'waiting': {...}}}, {u'image': u'gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1', u'imageID': u'', u'lastState': {}, u'name': u'kubedns', u'ready': False, u'restartCount': 0, u'state': {u'waiting': {...}}}, {u'image': u'gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.1', u'imageID': u'', u'lastState': {}, u'name': u'sidecar', u'ready': False, u'restartCount': 0, u'state': {u'waiting': {...}}}], u'hostIP': u'10.204.217.198', u'phase': u'Pending', u'qosClass': u'Burstable', u'startTime': u'2017-05-16T05:15:51Z'}}, u'type': u'ADDED'}
469 if event['type'] == 'ADDED':
470 vm = self.vnc_pod_add(pod_id, pod_name, pod_namespace,
471 pod_node, labels, vm_vmi)
472 if vm:
473 self._network_policy_mgr.update_pod_np(pod_namespace, pod_id, labels)
pod_node = u'testbed-1-vm3'
labels = {u'k8s-app': u'kube-dns', 'namespace': None, u'pod-template-hash': u'3121064917'}
vm_vmi = None
/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py in vnc_pod_add(self=<kube_manager.vnc.vnc_pod.VncPod object>, pod_id=u'b78011a8-39f6-11e7-81a4-525400010001', pod_name=u'kube-dns-3121064917-4t42w', pod_namespace=u'kube-system', pod_node=u'testbed-1-vm3', labels={u'k8s-app': u'kube-dns', 'namespace': None, u'pod-template-hash': u'3121064917'}, vm_vmi=None)
309 vm = VirtualMachineKM.get(pod_id)
310 if vm:
311 self._set_label_to_pod_cache(labels, vm)
312 return vm
313 else:
self = <kube_manager.vnc.vnc_pod.VncPod object>
self._set_label_to_pod_cache = <bound method VncPod._set_label_to_pod_cache of <kube_manager.vnc.vnc_pod.VncPod object>>
labels = {u'k8s-app': u'kube-dns', 'namespace': None, u'pod-template-hash': u'3121064917'}
vm = <kube_manager.vnc.config_db.VirtualMachineKM object>
/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py in _set_label_to_pod_cache(self=<kube_manager.vnc.vnc_pod.VncPod object>, new_labels={u'k8s-app': u'kube-dns', 'namespace': None, u'pod-template-hash': u'3121064917'}, vm=<kube_manager.vnc.config_db.VirtualMachineKM object>)
61 new_labels.update(namespace_label)
62 for label in new_labels.items():
63 key = self._label_cache._get_key(label)
64 self._label_cache._locate_label(key,
65 self._label_cache.pod_label_cache, label, vm.uuid)
key = u'k8s-app:kube-dns'
self = <kube_manager.vnc.vnc_pod.VncPod object>
self._label_cache = <kube_manager.vnc.label_cache.LabelCache object>
self._label_cache._get_key = <bound method LabelCache._get_key of <kube_manager.vnc.label_cache.LabelCache object>>
label = ('namespace', None)
/usr/lib/python2.7/dist-packages/kube_manager/vnc/label_cache.py in _get_key(self=<kube_manager.vnc.label_cache.LabelCache object>, label=('namespace', None))
11
12 def _get_key(self, label):
13 key = label[0] + ':' + label[1]
14 return key
15
key undefined
label = ('namespace', None)
<type 'exceptions.TypeError'>: cannot concatenate 'str' and 'NoneType' objects
__class__ = <type 'exceptions.TypeError'>
__delattr__ = <method-wrapper '__delattr__' of exceptions.TypeError object>
__dict__ = {}
__doc__ = 'Inappropriate argument type.'
__format__ = <built-in method __format__ of exceptions.TypeError object>
__getattribute__ = <method-wrapper '__getattribute__' of exceptions.TypeError object>
__getitem__ = <method-wrapper '__getitem__' of exceptions.TypeError object>
__getslice__ = <method-wrapper '__getslice__' of exceptions.TypeError object>
__hash__ = <method-wrapper '__hash__' of exceptions.TypeError object>
__init__ = <method-wrapper '__init__' of exceptions.TypeError object>
__new__ = <built-in method __new__ of type object>
__reduce__ = <built-in method __reduce__ of exceptions.TypeError object>
__reduce_ex__ = <built-in method __reduce_ex__ of exceptions.TypeError object>
__repr__ = <method-wrapper '__repr__' of exceptions.TypeError object>
__setattr__ = <method-wrapper '__setattr__' of exceptions.TypeError object>
__setstate__ = <built-in method __setstate__ of exceptions.TypeError object>
__sizeof__ = <built-in method __sizeof__ of exceptions.TypeError object>
__str__ = <method-wrapper '__str__' of exceptions.TypeError object>
__subclasshook__ = <built-in method __subclasshook__ of type object>
__unicode__ = <built-in method __unicode__ of exceptions.TypeError object>
args = ("cannot concatenate 'str' and 'NoneType' objects",)
message = "cannot concatenate 'str' and 'NoneType' objects"
The above is a description of an error in a Python program. Here is
the original traceback:
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_kubernetes.py", line 353, in vnc_process
self.pod_mgr.process(event)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 471, in process
pod_node, labels, vm_vmi)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 311, in vnc_pod_add
self._set_label_to_pod_cache(labels, vm)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 63, in _set_label_to_pod_cache
key = self._label_cache._get_key(label)
File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/label_cache.py", line 13, in _get_key
key = label[0] + ':' + label[1]
TypeError: cannot concatenate 'str' and 'NoneType' objects
Review in progress for https:/
Submitter: Yuvaraja Mariappan