As of 5022 MAAS is broken and constantly raising exceptions
==> logs/regiond2/current <==
2016-05-13 19:30:59 [-] Unhandled failure dispatching AMP command. This is probably a bug. Please ensure that this error is handled within application code or declared in the signature of the b'RegisterRackController' command. [maas-dev:pid=23625:cmd=RegisterRackController:ask=2]
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1184, in gotResult
_inlineCallbacks(r, g, deferred)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1174, in _inlineCallbacks
deferred.errback()
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 434, in errback
self._startRunCallbacks(fail)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 588, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/protocols/amp.py", line 1149, in checkKnownErrors
key = error.trap(*command.allErrors)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 342, in trap
self.raiseException()
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 368, in raiseException
raise self.value.with_traceback(self.tb)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/home/lee/maas/trunk/src/maasserver/rpc/regionservice.py", line 581, in register
unregisterConnection, region_id, self.ident, self.host)
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 246, in inContext
result = inContext.theWork()
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 262, in <lambda>
inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/home/lee/maas/trunk/src/provisioningserver/utils/twisted.py", line 937, in callInContext
return func(*args, **kwargs)
File "/home/lee/maas/trunk/src/maasserver/utils/orm.py", line 516, in call_within_transaction
return func_outside_txn(*args, **kwargs)
File "/home/lee/maas/trunk/src/maasserver/utils/orm.py", line 343, in retrier
return func(*args, **kwargs)
File "/usr/lib/python3.5/contextlib.py", line 30, in inner
return func(*args, **kwds)
File "/home/lee/maas/trunk/src/maasserver/rpc/regionservice.py", line 475, in unregisterConnection
region = RegionController.objects.get(system_id=region_id)
File "/usr/lib/python3/dist-packages/django/db/models/manager.py", line 127, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/lib/python3/dist-packages/django/db/models/query.py", line 334, in get
self.model._meta.object_name
maasserver.models.node.DoesNotExist: RegionController matching query does not exist.
==> logs/rackd/current <==
2016-05-13 12:30:59-0700 [ClusterClient,client] Event-loop 'maas-dev:pid=23625' handshake failed; dropping connection.
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 434, in errback
self._startRunCallbacks(fail)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks
self._runCallbacks()
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 588, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1184, in gotResult
_inlineCallbacks(r, g, deferred)
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/home/lee/maas/trunk/src/provisioningserver/rpc/clusterservice.py", line 593, in performHandshake
registered = yield self.registerRackWithRegion()
twisted.protocols.amp.UnhandledCommand: (b'UNHANDLED', 'Unknown Error [maas-dev:pid=23625:cmd=RegisterRackController:ask=2]')
2016-05-13 12:30:59-0700 [ClusterClient,client] ClusterClient connection lost (HOST:IPv4Address(TCP, '10.0.0.3', 47994) PEER:IPv4Address(TCP, '10.0.0.3', 5251))
==> logs/regiond2/current <==
2016-05-13 19:30:59 [RegionServer,2,10.0.0.3] Unhandled Error
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 285, in runReactorWithLogging
reactor.run()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1194, in run
self.mainLoop()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1206, in mainLoop
self.doIteration(t)
File "/usr/lib/python3/dist-packages/twisted/internet/epollreactor.py", line 396, in doPoll
log.callWithLogger(selectable, _drdw, selectable, fd, event)
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/python/log.py", line 101, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/log.py", line 84, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/usr/lib/python3/dist-packages/twisted/internet/posixbase.py", line 610, in _doReadOrWrite
self._disconnectSelectable(selectable, why, inRead)
File "/usr/lib/python3/dist-packages/twisted/internet/posixbase.py", line 252, in _disconnectSelectable
selectable.readConnectionLost(f)
File "/usr/lib/python3/dist-packages/twisted/internet/tcp.py", line 273, in readConnectionLost
self.connectionLost(reason)
File "/usr/lib/python3/dist-packages/twisted/internet/tcp.py", line 293, in connectionLost
protocol.connectionLost(reason)
File "/home/lee/maas/trunk/src/maasserver/rpc/regionservice.py", line 629, in connectionLost
self.getRegionID().addCallback(
builtins.AttributeError: 'NoneType' object has no attribute 'addCallback'
==> logs/regiond/current <==
2016-05-13 19:30:59 [-] Unhandled failure dispatching AMP command. This is probably a bug. Please ensure that this error is handled within application code or declared in the signature of the b'RegisterRackController' command. [maas-dev:pid=23553:cmd=RegisterRackController:ask=2]
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1184, in gotResult
_inlineCallbacks(r, g, deferred)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1174, in _inlineCallbacks
deferred.errback()
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 434, in errback
self._startRunCallbacks(fail)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 588, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/protocols/amp.py", line 1149, in checkKnownErrors
key = error.trap(*command.allErrors)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 342, in trap
self.raiseException()
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 368, in raiseException
raise self.value.with_traceback(self.tb)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/home/lee/maas/trunk/src/maasserver/rpc/regionservice.py", line 581, in register
unregisterConnection, region_id, self.ident, self.host)
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 246, in inContext
result = inContext.theWork()
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 262, in <lambda>
inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/home/lee/maas/trunk/src/provisioningserver/utils/twisted.py", line 937, in callInContext
return func(*args, **kwargs)
File "/home/lee/maas/trunk/src/maasserver/utils/orm.py", line 516, in call_within_transaction
return func_outside_txn(*args, **kwargs)
File "/home/lee/maas/trunk/src/maasserver/utils/orm.py", line 343, in retrier
return func(*args, **kwargs)
File "/usr/lib/python3.5/contextlib.py", line 30, in inner
return func(*args, **kwds)
File "/home/lee/maas/trunk/src/maasserver/rpc/regionservice.py", line 475, in unregisterConnection
region = RegionController.objects.get(system_id=region_id)
File "/usr/lib/python3/dist-packages/django/db/models/manager.py", line 127, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/lib/python3/dist-packages/django/db/models/query.py", line 334, in get
self.model._meta.object_name
maasserver.models.node.DoesNotExist: RegionController matching query does not exist.
==> logs/rackd/current <==
2016-05-13 12:30:59-0700 [ClusterClient,client] Event-loop 'maas-dev:pid=23553' handshake failed; dropping connection.
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 434, in errback
self._startRunCallbacks(fail)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks
self._runCallbacks()
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 588, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1184, in gotResult
_inlineCallbacks(r, g, deferred)
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/home/lee/maas/trunk/src/provisioningserver/rpc/clusterservice.py", line 593, in performHandshake
registered = yield self.registerRackWithRegion()
twisted.protocols.amp.UnhandledCommand: (b'UNHANDLED', 'Unknown Error [maas-dev:pid=23553:cmd=RegisterRackController:ask=2]')
2016-05-13 12:30:59-0700 [ClusterClient,client] ClusterClient connection lost (HOST:IPv4Address(TCP, '10.0.0.3', 48430) PEER:IPv4Address(TCP, '10.0.0.3', 5250))
==> logs/regiond/current <==
2016-05-13 19:30:59 [RegionServer,1,10.0.0.3] Unhandled Error
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 285, in runReactorWithLogging
reactor.run()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1194, in run
self.mainLoop()
File "/usr/lib/python3/dist-packages/twisted/internet/base.py", line 1206, in mainLoop
self.doIteration(t)
File "/usr/lib/python3/dist-packages/twisted/internet/epollreactor.py", line 396, in doPoll
log.callWithLogger(selectable, _drdw, selectable, fd, event)
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/python/log.py", line 101, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/log.py", line 84, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/usr/lib/python3/dist-packages/twisted/internet/posixbase.py", line 610, in _doReadOrWrite
self._disconnectSelectable(selectable, why, inRead)
File "/usr/lib/python3/dist-packages/twisted/internet/posixbase.py", line 252, in _disconnectSelectable
selectable.readConnectionLost(f)
File "/usr/lib/python3/dist-packages/twisted/internet/tcp.py", line 273, in readConnectionLost
self.connectionLost(reason)
File "/usr/lib/python3/dist-packages/twisted/internet/tcp.py", line 293, in connectionLost
protocol.connectionLost(reason)
File "/home/lee/maas/trunk/src/maasserver/rpc/regionservice.py", line 629, in connectionLost
self.getRegionID().addCallback(