I've dropped all "b" prefixes from the power states in provisioningserver/drivers/power/amt.py and now I'm seeing another error, which causes a similar power check errors. The new error is because apparently the AMT controller often responds with "500 Can't connect to <IP>", which is treated as PowerFatalError, but it shouldn't (as retrying usually works as expected - in my observations at least).
The error I described earlier causes the following traceback in the rackd.log:
2016-04-13 16:22:15+0300 [ClusterClient,client] Failed to refresh power state.
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 434, in errback self._startRunCallbacks(fail)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks self._runCallbacks()
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 588, in _runCallbacks current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1184, in gotResult _inlineCallbacks(r, g, deferred)
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/python3/dist-packages/provisioningserver/power/query.py", line 103, in get_power_state
raise exc_type(exc_value).with_traceback(exc_trace)
File "/usr/lib/python3/dist-packages/provisioningserver/power/query.py", line 92, in get_power_state system_id, hostname, power_type, context)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1126, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/python3/dist-packages/provisioningserver/drivers/power/__init__.py", line 240, in query self.power_query, system_id, context)
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 246, in inContext
result = inContext.theWork()
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 262, in <lambda> inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
File "/usr/lib/python3/dist-packages/provisioningserver/drivers/power/amt.py", line 367, in power_query amt_command = self._get_amt_command(ip_address, power_pass)
File "/usr/lib/python3/dist-packages/provisioningserver/drivers/power/amt.py", line 293, in _get_amt_command
"Unable to retrieve AMT version: %s" % stderr) provisioningserver.drivers.power.PowerFatalError: Unable to retrieve AMT version: 500 Can't connect to 10.14.0.12:16992 at /usr/bin/amttool line 126.
I've dropped all "b" prefixes from the power states in provisioningser ver/drivers/ power/amt. py and now I'm seeing another error, which causes a similar power check errors. The new error is because apparently the AMT controller often responds with "500 Can't connect to <IP>", which is treated as PowerFatalError, but it shouldn't (as retrying usually works as expected - in my observations at least).
Here's the patch I used: http:// paste.ubuntu. com/15810913/
The error I described earlier causes the following traceback in the rackd.log:
2016-04-13 16:22:15+0300 [ClusterClient, client] Failed to refresh power state. python3/ dist-packages/ twisted/ internet/ defer.py" , line 434, in errback
self. _startRunCallba cks(fail) python3/ dist-packages/ twisted/ internet/ defer.py" , line 501, in _startRunCallbacks
self. _runCallbacks( ) python3/ dist-packages/ twisted/ internet/ defer.py" , line 588, in _runCallbacks
current. result = callback( current. result, *args, **kw) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1184, in gotResult
_inlineCal lbacks( r, g, deferred) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1126, in _inlineCallbacks throwExceptionI ntoGenerator( g) python3/ dist-packages/ twisted/ python/ failure. py", line 389, in throwExceptionI ntoGenerator python3/ dist-packages/ provisioningser ver/power/ query.py" , line 103, in get_power_state exc_value) .with_traceback (exc_trace) python3/ dist-packages/ provisioningser ver/power/ query.py" , line 92, in get_power_state
system_ id, hostname, power_type, context) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1126, in _inlineCallbacks throwExceptionI ntoGenerator( g) python3/ dist-packages/ twisted/ python/ failure. py", line 389, in throwExceptionI ntoGenerator python3/ dist-packages/ provisioningser ver/drivers/ power/_ _init__ .py", line 240, in query
self. power_query, system_id, context) python3/ dist-packages/ twisted/ python/ threadpool. py", line 246, in inContext python3/ dist-packages/ twisted/ python/ threadpool. py", line 262, in <lambda>
inContext. theWork = lambda: context.call(ctx, func, *args, **kw) python3/ dist-packages/ twisted/ python/ context. py", line 118, in callWithContext text(). callWithContext (ctx, func, *args, **kw) python3/ dist-packages/ twisted/ python/ context. py", line 81, in callWithContext python3/ dist-packages/ provisioningser ver/drivers/ power/amt. py", line 367, in power_query
amt_ command = self._get_ amt_command( ip_address, power_pass) python3/ dist-packages/ provisioningser ver/drivers/ power/amt. py", line 293, in _get_amt_command
provisioningse rver.drivers. power.PowerFata lError: Unable to retrieve AMT version: 500 Can't connect to 10.14.0.12:16992 at /usr/bin/amttool line 126.
Traceback (most recent call last):
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
--- <exception caught here> ---
File "/usr/lib/
result = result.
File "/usr/lib/
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/
raise exc_type(
File "/usr/lib/
File "/usr/lib/
result = result.
File "/usr/lib/
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/
File "/usr/lib/
result = inContext.theWork()
File "/usr/lib/
File "/usr/lib/
return self.currentCon
File "/usr/lib/
return func(*args,**kw)
File "/usr/lib/
File "/usr/lib/
"Unable to retrieve AMT version: %s" % stderr)