The issue has reproduced again - here's a backtrace I was able to capture from the hung regiond process:
Thread 0x00007fa414f8d700 (most recent call first):
File "/usr/lib/python3.8/threading.py", line 306 in wait
File "/usr/lib/python3.8/threading.py", line 558 in wait
File "/usr/lib/python3/dist-packages/crochet/_eventloop.py", line 189 in _result
File "/usr/lib/python3/dist-packages/crochet/_eventloop.py", line 229 in wait
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 126 in wrapper
File "/usr/lib/python3/dist-packages/maasserver/bootresources.py", line 1359 in _import_resources_without_lock
File "/usr/lib/python3/dist-packages/maasserver/bootresources.py", line 1429 in _import_resources_with_lock
File "/usr/lib/python3/dist-packages/maasserver/utils/__init__.py", line 193 in call_with_lock
File "/usr/lib/python3/dist-packages/maasserver/utils/orm.py", line 692 in call_with_connection
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 192 in wrapper
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 860 in callInContext
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 85 in callWithContext
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 122 in callWithContext
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 266 in <lambda>
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 250 in inContext
File "/usr/lib/python3/dist-packages/twisted/_threads/_team.py", line 190 in doWork
File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 46 in work
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
File "/usr/lib/python3.8/threading.py", line 870 in run
File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007fa415bdd700 (most recent call first):
File "/usr/lib/python3.8/threading.py", line 302 in wait
File "/usr/lib/python3.8/queue.py", line 170 in get
File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
File "/usr/lib/python3.8/threading.py", line 870 in run
File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007fa41641e700 (most recent call first):
File "/usr/lib/python3.8/threading.py", line 302 in wait
File "/usr/lib/python3.8/queue.py", line 170 in get
File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
File "/usr/lib/python3.8/threading.py", line 870 in run
File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007fa416c5f700 (most recent call first):
File "/usr/lib/python3.8/threading.py", line 302 in wait
File "/usr/lib/python3.8/queue.py", line 170 in get
File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
File "/usr/lib/python3.8/threading.py", line 870 in run
File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007fa417460700 (most recent call first):
File "/usr/lib/python3.8/threading.py", line 302 in wait
File "/usr/lib/python3.8/queue.py", line 170 in get
File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
File "/usr/lib/python3.8/threading.py", line 870 in run
File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007fa417db5700 (most recent call first):
File "/usr/lib/python3.8/threading.py", line 302 in wait
File "/usr/lib/python3.8/queue.py", line 170 in get
File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 45 in work
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
File "/usr/lib/python3.8/threading.py", line 870 in run
File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007fa4185b6700 (most recent call first):
File "/usr/lib/python3.8/threading.py", line 306 in wait
File "/usr/lib/python3.8/threading.py", line 558 in wait
File "/usr/lib/python3/dist-packages/crochet/_eventloop.py", line 189 in _result
File "/usr/lib/python3/dist-packages/crochet/_eventloop.py", line 229 in wait
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 126 in wrapper
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 192 in wrapper
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 860 in callInContext
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 85 in callWithContext
File "/usr/lib/python3/dist-packages/twisted/python/context.py", line 122 in callWithContext
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 266 in <lambda>
File "/usr/lib/python3/dist-packages/twisted/python/threadpool.py", line 250 in inContext
File "/usr/lib/python3/dist-packages/twisted/_threads/_team.py", line 190 in doWork
File "/usr/lib/python3/dist-packages/twisted/_threads/_threadworker.py", line 46 in work
File "/usr/lib/python3/dist-packages/provisioningserver/utils/twisted.py", line 825 in worker
File "/usr/lib/python3.8/threading.py", line 870 in run
File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap
Current thread 0x00007fa420670740 (most recent call first):
File "/usr/lib/python3/dist-packages/OpenSSL/SSL.py", line 1914 in do_handshake
File "/usr/lib/python3/dist-packages/urllib3/contrib/pyopenssl.py", line 485 in wrap_socket
File "/usr/lib/python3/dist-packages/urllib3/util/ssl_.py", line 370 in ssl_wrap_socket
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 366 in connect
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 996 in _validate_conn
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 376 in _make_request
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 665 in urlopen
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 439 in send
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 646 in send
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 533 in request
File "/usr/lib/python3/dist-packages/requests_unixsocket/__init__.py", line 46 in request
File "/usr/lib/python3/dist-packages/requests_unixsocket/__init__.py", line 51 in get
File "/usr/lib/python3/dist-packages/simplestreams/contentsource.py", line 381 in __init__
File "/usr/lib/python3/dist-packages/simplestreams/mirrors/__init__.py", line 186 in url_reader_factory
File "/usr/lib/python3/dist-packages/simplestreams/contentsource.py", line 119 in _open
File "/usr/lib/python3/dist-packages/simplestreams/contentsource.py", line 131 in open
File "/usr/lib/python3/dist-packages/simplestreams/contentsource.py", line 74 in __enter__
File "/usr/lib/python3/dist-packages/simplestreams/mirrors/__init__.py", line 41 in read_json
File "/usr/lib/python3/dist-packages/simplestreams/mirrors/__init__.py", line 85 in sync
File "/usr/lib/python3/dist-packages/provisioningserver/import_images/download_descriptions.py", line 201 in sync
File "/usr/lib/python3/dist-packages/provisioningserver/import_images/download_descriptions.py", line 333 in download_image_descriptions
File "/usr/lib/python3/dist-packages/provisioningserver/import_images/download_descriptions.py", line 343 in download_all_image_descriptions
File "/usr/lib/python3/dist-packages/maasserver/bootsources.py", line 360 in cache_boot_sources
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1418 in _inlineCallbacks
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1475 in gotResult
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654 in _runCallbacks
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 568 in _startRunCallbacks
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 460 in callback
File "/usr/lib/python3/dist-packages/twisted/internet/asyncioreactor.py", line 290 in run
File "/usr/lib/python3/dist-packages/twisted/internet/asyncioreactor.py", line 267 in run
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 312 in runReactorWithLogging
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 398 in startReactor
File "/usr/lib/python3/dist-packages/twisted/scripts/_twistd_unix.py", line 268 in postApplication
File "/usr/lib/python3/dist-packages/twisted/application/app.py", line 385 in run
File "/usr/lib/python3/dist-packages/provisioningserver/server.py", line 90 in runService
File "/usr/lib/python3/dist-packages/maasserver/server.py", line 35 in runWorkerServices
File "/usr/lib/python3/dist-packages/maasserver/server.py", line 78 in run
File "/usr/sbin/regiond", line 11 in <module>
The issue has reproduced again - here's a backtrace I was able to capture from the hung regiond process:
Thread 0x00007fa414f8d700 (most recent call first): python3. 8/threading. py", line 306 in wait python3. 8/threading. py", line 558 in wait python3/ dist-packages/ crochet/ _eventloop. py", line 189 in _result python3/ dist-packages/ crochet/ _eventloop. py", line 229 in wait python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 126 in wrapper python3/ dist-packages/ maasserver/ bootresources. py", line 1359 in _import_ resources_ without_ lock python3/ dist-packages/ maasserver/ bootresources. py", line 1429 in _import_ resources_ with_lock python3/ dist-packages/ maasserver/ utils/_ _init__ .py", line 193 in call_with_lock python3/ dist-packages/ maasserver/ utils/orm. py", line 692 in call_with_ connection python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 192 in wrapper python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 860 in callInContext python3/ dist-packages/ twisted/ python/ context. py", line 85 in callWithContext python3/ dist-packages/ twisted/ python/ context. py", line 122 in callWithContext python3/ dist-packages/ twisted/ python/ threadpool. py", line 266 in <lambda> python3/ dist-packages/ twisted/ python/ threadpool. py", line 250 in inContext python3/ dist-packages/ twisted/ _threads/ _team.py" , line 190 in doWork python3/ dist-packages/ twisted/ _threads/ _threadworker. py", line 46 in work python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 825 in worker python3. 8/threading. py", line 870 in run python3. 8/threading. py", line 932 in _bootstrap_inner python3. 8/threading. py", line 890 in _bootstrap
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
Thread 0x00007fa415bdd700 (most recent call first): python3. 8/threading. py", line 302 in wait python3. 8/queue. py", line 170 in get python3/ dist-packages/ twisted/ _threads/ _threadworker. py", line 45 in work python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 825 in worker python3. 8/threading. py", line 870 in run python3. 8/threading. py", line 932 in _bootstrap_inner python3. 8/threading. py", line 890 in _bootstrap
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
Thread 0x00007fa41641e700 (most recent call first): python3. 8/threading. py", line 302 in wait python3. 8/queue. py", line 170 in get python3/ dist-packages/ twisted/ _threads/ _threadworker. py", line 45 in work python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 825 in worker python3. 8/threading. py", line 870 in run python3. 8/threading. py", line 932 in _bootstrap_inner python3. 8/threading. py", line 890 in _bootstrap
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
Thread 0x00007fa416c5f700 (most recent call first): python3. 8/threading. py", line 302 in wait python3. 8/queue. py", line 170 in get python3/ dist-packages/ twisted/ _threads/ _threadworker. py", line 45 in work python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 825 in worker python3. 8/threading. py", line 870 in run python3. 8/threading. py", line 932 in _bootstrap_inner python3. 8/threading. py", line 890 in _bootstrap
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
Thread 0x00007fa417460700 (most recent call first): python3. 8/threading. py", line 302 in wait python3. 8/queue. py", line 170 in get python3/ dist-packages/ twisted/ _threads/ _threadworker. py", line 45 in work python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 825 in worker python3. 8/threading. py", line 870 in run python3. 8/threading. py", line 932 in _bootstrap_inner python3. 8/threading. py", line 890 in _bootstrap
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
Thread 0x00007fa417db5700 (most recent call first): python3. 8/threading. py", line 302 in wait python3. 8/queue. py", line 170 in get python3/ dist-packages/ twisted/ _threads/ _threadworker. py", line 45 in work python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 825 in worker python3. 8/threading. py", line 870 in run python3. 8/threading. py", line 932 in _bootstrap_inner python3. 8/threading. py", line 890 in _bootstrap
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
Thread 0x00007fa4185b6700 (most recent call first): python3. 8/threading. py", line 306 in wait python3. 8/threading. py", line 558 in wait python3/ dist-packages/ crochet/ _eventloop. py", line 189 in _result python3/ dist-packages/ crochet/ _eventloop. py", line 229 in wait python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 126 in wrapper python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 192 in wrapper python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 860 in callInContext python3/ dist-packages/ twisted/ python/ context. py", line 85 in callWithContext python3/ dist-packages/ twisted/ python/ context. py", line 122 in callWithContext python3/ dist-packages/ twisted/ python/ threadpool. py", line 266 in <lambda> python3/ dist-packages/ twisted/ python/ threadpool. py", line 250 in inContext python3/ dist-packages/ twisted/ _threads/ _team.py" , line 190 in doWork python3/ dist-packages/ twisted/ _threads/ _threadworker. py", line 46 in work python3/ dist-packages/ provisioningser ver/utils/ twisted. py", line 825 in worker python3. 8/threading. py", line 870 in run python3. 8/threading. py", line 932 in _bootstrap_inner python3. 8/threading. py", line 890 in _bootstrap
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
Current thread 0x00007fa420670740 (most recent call first): python3/ dist-packages/ OpenSSL/ SSL.py" , line 1914 in do_handshake python3/ dist-packages/ urllib3/ contrib/ pyopenssl. py", line 485 in wrap_socket python3/ dist-packages/ urllib3/ util/ssl_ .py", line 370 in ssl_wrap_socket python3/ dist-packages/ urllib3/ connection. py", line 366 in connect python3/ dist-packages/ urllib3/ connectionpool. py", line 996 in _validate_conn python3/ dist-packages/ urllib3/ connectionpool. py", line 376 in _make_request python3/ dist-packages/ urllib3/ connectionpool. py", line 665 in urlopen python3/ dist-packages/ requests/ adapters. py", line 439 in send python3/ dist-packages/ requests/ sessions. py", line 646 in send python3/ dist-packages/ requests/ sessions. py", line 533 in request python3/ dist-packages/ requests_ unixsocket/ __init_ _.py", line 46 in request python3/ dist-packages/ requests_ unixsocket/ __init_ _.py", line 51 in get python3/ dist-packages/ simplestreams/ contentsource. py", line 381 in __init__ python3/ dist-packages/ simplestreams/ mirrors/ __init_ _.py", line 186 in url_reader_factory python3/ dist-packages/ simplestreams/ contentsource. py", line 119 in _open python3/ dist-packages/ simplestreams/ contentsource. py", line 131 in open python3/ dist-packages/ simplestreams/ contentsource. py", line 74 in __enter__ python3/ dist-packages/ simplestreams/ mirrors/ __init_ _.py", line 41 in read_json python3/ dist-packages/ simplestreams/ mirrors/ __init_ _.py", line 85 in sync python3/ dist-packages/ provisioningser ver/import_ images/ download_ descriptions. py", line 201 in sync python3/ dist-packages/ provisioningser ver/import_ images/ download_ descriptions. py", line 333 in download_ image_descripti ons python3/ dist-packages/ provisioningser ver/import_ images/ download_ descriptions. py", line 343 in download_ all_image_ descriptions python3/ dist-packages/ maasserver/ bootsources. py", line 360 in cache_boot_sources python3/ dist-packages/ twisted/ internet/ defer.py" , line 1418 in _inlineCallbacks python3/ dist-packages/ twisted/ internet/ defer.py" , line 1475 in gotResult python3/ dist-packages/ twisted/ internet/ defer.py" , line 654 in _runCallbacks python3/ dist-packages/ twisted/ internet/ defer.py" , line 568 in _startRunCallbacks python3/ dist-packages/ twisted/ internet/ defer.py" , line 460 in callback python3/ dist-packages/ twisted/ internet/ asyncioreactor. py", line 290 in run python3/ dist-packages/ twisted/ internet/ asyncioreactor. py", line 267 in run python3/ dist-packages/ twisted/ application/ app.py" , line 312 in runReactorWithL ogging python3/ dist-packages/ twisted/ application/ app.py" , line 398 in startReactor python3/ dist-packages/ twisted/ scripts/ _twistd_ unix.py" , line 268 in postApplication python3/ dist-packages/ twisted/ application/ app.py" , line 385 in run python3/ dist-packages/ provisioningser ver/server. py", line 90 in runService python3/ dist-packages/ maasserver/ server. py", line 35 in runWorkerServices python3/ dist-packages/ maasserver/ server. py", line 78 in run regiond" , line 11 in <module>
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/lib/
File "/usr/sbin/