We are currently this with a 2.7.0 release on a node with LANG=en_US, which hasn't changed since the deployment of the last version we ran. However we see it in a much more problematic case: if a single object-server has services offline or is down, and Swift tries to log the corresponding error for it, any user-facing PUT request that attempts to touch that node fails with a 500, rather than gracefully trying a handoff node.
swift-proxy: ERROR Unhandled exception in request:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/swift/proxy/server.py", line 411, in handle_request
return handler(req)
File "/usr/local/lib/python2.7/dist-packages/swift/proxy/controllers/base.py", line 264, in wrapped
return func(*a, **kw)
File "/usr/local/lib/python2.7/dist-packages/swift/proxy/controllers/obj.py", line 775, in PUT
req, data_source, nodes, partition, outgoing_headers)
File "/usr/local/lib/python2.7/dist-packages/swift/proxy/controllers/obj.py", line 1048, in _store_object
outgoing_headers, policy, expect)
File "/usr/local/lib/python2.7/dist-packages/swift/proxy/controllers/obj.py", line 639, in _get_put_connections
conns = [conn for conn in pile if conn]
File "/usr/local/lib/python2.7/dist-packages/eventlet/greenpool.py", line 222, in next
return self.waiters.get().wait()
File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 175, in wait
return self._exit_event.wait()
File "/usr/local/lib/python2.7/dist-packages/eventlet/event.py", line 121, in wait
return hubs.get_hub().switch()
File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 294, in switch
return self.greenlet.switch()
File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 214, in main
result = function(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/swift/proxy/controllers/obj.py", line 935, in _connect_put_node
_('Expect: 100-continue on %s') % path)
File "/usr/local/lib/python2.7/dist-packages/swift/proxy/server.py", line 536, in exception_occurred
**kwargs)
File "/usr/local/lib/python2.7/dist-packages/swift/common/utils.py", line 1513, in exception
call('%s: %s' % (msg, emsg), *args, **kwargs)
File "/usr/lib/python2.7/logging/__init__.py", line 1445, in error
self.logger.error(msg, *args, **kwargs)
File "/usr/lib/python2.7/logging/__init__.py", line 1174, in error
self._log(ERROR, msg, args, **kwargs)
File "/usr/lib/python2.7/logging/__init__.py", line 1267, in _log
self.handle(record)
File "/usr/lib/python2.7/logging/__init__.py", line 1277, in handle
self.callHandlers(record)
File "/usr/lib/python2.7/logging/__init__.py", line 1317, in callHandlers
hdlr.handle(record)
File "/usr/lib/python2.7/logging/__init__.py", line 748, in handle
self.emit(record)
File "/usr/lib/python2.7/logging/handlers.py", line 828, in emit
msg = self.format(record) + '\000'
File "/usr/lib/python2.7/logging/__init__.py", line 723, in format
return fmt.format(record)
File "/usr/local/lib/python2.7/dist-packages/swift/common/utils.py", line 1571, in format
record.message = record.getMessage()
File "/usr/lib/python2.7/logging/__init__.py", line 328, in getMessage
msg = msg % self.args
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe5 in position 54: ordinal not in range(128) (txn: txb9ca722577604d2f9f959-00577190f3) (client_ip: 127.0.0.1)
We are currently this with a 2.7.0 release on a node with LANG=en_US, which hasn't changed since the deployment of the last version we ran. However we see it in a much more problematic case: if a single object-server has services offline or is down, and Swift tries to log the corresponding error for it, any user-facing PUT request that attempts to touch that node fails with a 500, rather than gracefully trying a handoff node.
Un-mangled stack trace is at https:/ /gist.github. com/briancline/ 82bef904104d21e 91db7829f5d8fc1 92
swift-proxy: ERROR Unhandled exception in request: lib/python2. 7/dist- packages/ swift/proxy/ server. py", line 411, in handle_request lib/python2. 7/dist- packages/ swift/proxy/ controllers/ base.py" , line 264, in wrapped lib/python2. 7/dist- packages/ swift/proxy/ controllers/ obj.py" , line 775, in PUT lib/python2. 7/dist- packages/ swift/proxy/ controllers/ obj.py" , line 1048, in _store_object headers, policy, expect) lib/python2. 7/dist- packages/ swift/proxy/ controllers/ obj.py" , line 639, in _get_put_ connections lib/python2. 7/dist- packages/ eventlet/ greenpool. py", line 222, in next get().wait( ) lib/python2. 7/dist- packages/ eventlet/ greenthread. py", line 175, in wait event.wait( ) lib/python2. 7/dist- packages/ eventlet/ event.py" , line 121, in wait hub().switch( ) lib/python2. 7/dist- packages/ eventlet/ hubs/hub. py", line 294, in switch switch( ) lib/python2. 7/dist- packages/ eventlet/ greenthread. py", line 214, in main lib/python2. 7/dist- packages/ swift/proxy/ controllers/ obj.py" , line 935, in _connect_put_node lib/python2. 7/dist- packages/ swift/proxy/ server. py", line 536, in exception_occurred lib/python2. 7/dist- packages/ swift/common/ utils.py" , line 1513, in exception python2. 7/logging/ __init_ _.py", line 1445, in error logger. error(msg, *args, **kwargs) python2. 7/logging/ __init_ _.py", line 1174, in error _log(ERROR, msg, args, **kwargs) python2. 7/logging/ __init_ _.py", line 1267, in _log handle( record) python2. 7/logging/ __init_ _.py", line 1277, in handle callHandlers( record) python2. 7/logging/ __init_ _.py", line 1317, in callHandlers handle( record) python2. 7/logging/ __init_ _.py", line 748, in handle emit(record) python2. 7/logging/ handlers. py", line 828, in emit python2. 7/logging/ __init_ _.py", line 723, in format lib/python2. 7/dist- packages/ swift/common/ utils.py" , line 1571, in format python2. 7/logging/ __init_ _.py", line 328, in getMessage deError: 'ascii' codec can't decode byte 0xe5 in position 54: ordinal not in range(128) (txn: txb9ca722577604 d2f9f959- 00577190f3) (client_ip: 127.0.0.1)
Traceback (most recent call last):
File "/usr/local/
return handler(req)
File "/usr/local/
return func(*a, **kw)
File "/usr/local/
req, data_source, nodes, partition, outgoing_headers)
File "/usr/local/
outgoing_
File "/usr/local/
conns = [conn for conn in pile if conn]
File "/usr/local/
return self.waiters.
File "/usr/local/
return self._exit_
File "/usr/local/
return hubs.get_
File "/usr/local/
return self.greenlet.
File "/usr/local/
result = function(*args, **kwargs)
File "/usr/local/
_('Expect: 100-continue on %s') % path)
File "/usr/local/
**kwargs)
File "/usr/local/
call('%s: %s' % (msg, emsg), *args, **kwargs)
File "/usr/lib/
self.
File "/usr/lib/
self.
File "/usr/lib/
self.
File "/usr/lib/
self.
File "/usr/lib/
hdlr.
File "/usr/lib/
self.
File "/usr/lib/
msg = self.format(record) + '\000'
File "/usr/lib/
return fmt.format(record)
File "/usr/local/
record.message = record.getMessage()
File "/usr/lib/
msg = msg % self.args
UnicodeDeco