intermittent timeout in worker/raft/raftclusterer tests

Bug #1771477 reported by Christian Muirhead
6
This bug affects 1 person
Affects Status Importance Assigned to Milestone
Canonical Juju
Fix Released
Medium
Christian Muirhead

Bug Description

As seen here:

http://ci.jujucharms.com/job/github-merge-juju/456/console

=== RUN TestPackage
panic: test timed out after 25m0s

goroutine 274 [running]:
testing.(*M).startAlarm.func1()
 /snap/go/1880/src/testing/testing.go:1240 +0xfc
created by time.goFunc
 /snap/go/1880/src/time/sleep.go:172 +0x44

goroutine 1 [chan receive, 25 minutes]:
testing.(*T).Run(0xc4201be0f0, 0xb787b4, 0xb, 0xba6ea8, 0x480146)
 /snap/go/1880/src/testing/testing.go:825 +0x301
testing.runTests.func1(0xc4201be000)
 /snap/go/1880/src/testing/testing.go:1063 +0x64
testing.tRunner(0xc4201be000, 0xc4201a5df8)
 /snap/go/1880/src/testing/testing.go:777 +0xd0
testing.runTests(0xc42038c620, 0xf27420, 0x1, 0x1, 0x412209)
 /snap/go/1880/src/testing/testing.go:1061 +0x2c4
testing.(*M).Run(0xc420298380, 0x0)
 /snap/go/1880/src/testing/testing.go:978 +0x171
main.main()
 _testmain.go:44 +0x151

goroutine 5 [syscall, 25 minutes]:
os/signal.signal_recv(0x0)
 /snap/go/1880/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
 /snap/go/1880/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
 /snap/go/1880/src/os/signal/signal_unix.go:28 +0x41

goroutine 172 [semacquire, 24 minutes]:
sync.runtime_Semacquire(0xc4204c6b4c)
 /snap/go/1880/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc4204c6b40)
 /snap/go/1880/src/sync/waitgroup.go:129 +0x72
github.com/hashicorp/raft.(*raftState).waitShutdown(0xc4204c6b00)
 /workspace/src/github.com/hashicorp/raft/state.go:151 +0x31
github.com/hashicorp/raft.(*shutdownFuture).Error(0xc4203cc000, 0xbfaf20, 0xc4203cc000)
 /workspace/src/github.com/hashicorp/raft/future.go:158 +0x37
github.com/juju/juju/worker/raft/rafttest.(*RaftFixture).NewRaft.func2(0xc4201d8000)
 /workspace/src/github.com/juju/juju/worker/raft/rafttest/fixtures.go:70 +0x42
github.com/juju/testing.(*CleanupSuite).callStack(0xc4202b8008, 0xc4201d8000, 0xc4205fc140, 0x8, 0x8)
 /workspace/src/github.com/juju/testing/cleanup.go:54 +0x44
github.com/juju/testing.(*CleanupSuite).TearDownTest(0xc4202b8008, 0xc4201d8000)
 /workspace/src/github.com/juju/testing/cleanup.go:47 +0x4f
github.com/juju/testing.(*IsolationSuite).TearDownTest(0xc4202b8000, 0xc4201d8000)
 /workspace/src/github.com/juju/testing/isolation.go:38 +0x3b
reflect.Value.call(0xb58f00, 0xc4202b8000, 0x2a13, 0xb71e23, 0x4, 0xc420347f70, 0x1, 0x1, 0xc4203d2500, 0xb65aa0, ...)
 /snap/go/1880/src/reflect/value.go:447 +0x969
reflect.Value.Call(0xb58f00, 0xc4202b8000, 0x2a13, 0xc42035e770, 0x1, 0x1, 0xc42035e788, 0x42b7d9, 0xc420031e70)
 /snap/go/1880/src/reflect/value.go:308 +0xa4
gopkg.in/check%2ev1.(*suiteRunner).runFixture.func1(0xc4201d8000)
 /workspace/src/gopkg.in/check.v1/check.go:721 +0xe1
gopkg.in/check%2ev1.(*suiteRunner).forkCall.func1(0xc420298880, 0xc4201d8000, 0xba71e0)
 /workspace/src/gopkg.in/check.v1/check.go:666 +0x7c
created by gopkg.in/check%2ev1.(*suiteRunner).forkCall
 /workspace/src/gopkg.in/check.v1/check.go:663 +0x20c

goroutine 198 [select, 24 minutes]:
github.com/juju/pubsub.(*subscriber).loop(0xc4203853b0)
 /workspace/src/github.com/juju/pubsub/subscriber.go:59 +0x10d
created by github.com/juju/pubsub.newSubscriber
 /workspace/src/github.com/juju/pubsub/subscriber.go:40 +0x14d

goroutine 15 [chan receive, 24 minutes]:
gopkg.in/check%2ev1.(*suiteRunner).runTest(0xc420298880, 0xc4201bbab0, 0xc4201d84b0)
 /workspace/src/gopkg.in/check.v1/check.go:810 +0x55
gopkg.in/check%2ev1.(*suiteRunner).run(0xc420298880, 0xc4202b8000)
 /workspace/src/gopkg.in/check.v1/check.go:615 +0xf2
gopkg.in/check%2ev1.Run(0xb58f00, 0xc4202b8000, 0xc420294c00, 0xc4201c41b0)
 /workspace/src/gopkg.in/check.v1/run.go:92 +0x4d
gopkg.in/check%2ev1.RunAll(0xc420294c00, 0x0)
 /workspace/src/gopkg.in/check.v1/run.go:84 +0xa1
gopkg.in/check%2ev1.TestingT(0xc4201be0f0)
 /workspace/src/gopkg.in/check.v1/run.go:72 +0x368
github.com/juju/juju/worker/raft/raftclusterer_test.TestPackage(0xc4201be0f0)
 /workspace/src/github.com/juju/juju/worker/raft/raftclusterer/package_test.go:13 +0x2b
testing.tRunner(0xc4201be0f0, 0xba6ea8)
 /snap/go/1880/src/testing/testing.go:777 +0xd0
created by testing.(*T).Run
 /snap/go/1880/src/testing/testing.go:824 +0x2e0

goroutine 59 [select, 24 minutes]:
gopkg.in/check%2ev1.(*resultTracker)._loopRoutine(0xc4201c43f0)
 /workspace/src/gopkg.in/check.v1/check.go:461 +0x109
created by gopkg.in/check%2ev1.(*resultTracker).start
 /workspace/src/gopkg.in/check.v1/check.go:441 +0x3f

goroutine 202 [select, 24 minutes]:
github.com/juju/juju/worker/raft/raftclusterer.(*Worker).loop(0xc4201ba690, 0xc42028ba70, 0xc1f11d5133d28583)
 /workspace/src/github.com/juju/juju/worker/raft/raftclusterer/worker.go:109 +0x13b
github.com/juju/juju/worker/raft/raftclusterer.NewWorker.func1(0x0, 0x0)
 /workspace/src/github.com/juju/juju/worker/raft/raftclusterer/worker.go:73 +0x55
github.com/juju/juju/worker/catacomb.runSafely(0xc420203580, 0x0, 0x0)
 /workspace/src/github.com/juju/juju/worker/catacomb/catacomb.go:289 +0x55
github.com/juju/juju/worker/catacomb.Invoke.func3(0x0, 0x0)
 /workspace/src/github.com/juju/juju/worker/catacomb/catacomb.go:115 +0x70
gopkg.in/tomb%2ev2.(*Tomb).run(0xc4201ba690, 0xc4204b1780)
 /workspace/src/gopkg.in/tomb.v2/tomb.go:153 +0x2b
created by gopkg.in/tomb%2ev2.(*Tomb).Go
 /workspace/src/gopkg.in/tomb.v2/tomb.go:149 +0xb9

goroutine 125 [chan send, 24 minutes]:
github.com/hashicorp/raft.(*InmemTransport).makeRPC(0xc4206196e0, 0xc42060d800, 0x24, 0xa84f00, 0xc4204fb0e0, 0x0, 0x0, 0x2faf080, 0x5afb7899, 0xc42056bc88, ...)
 /workspace/src/github.com/hashicorp/raft/inmem_transport.go:143 +0x15d
github.com/hashicorp/raft.(*InmemTransport).AppendEntries(0xc4206196e0, 0xc420210e30, 0x9, 0xc42060d800, 0x24, 0xc4204fb0e0, 0xc42065c4a0, 0x2, 0x3)
 /workspace/src/github.com/hashicorp/raft/inmem_transport.go:94 +0x76
github.com/hashicorp/raft.(*Raft).replicateTo(0xc4204c6b00, 0xc4202b8180, 0x6, 0x0)
 /workspace/src/github.com/hashicorp/raft/replication.go:186 +0x189
github.com/hashicorp/raft.(*Raft).replicate(0xc4204c6b00, 0xc4202b8180)
 /workspace/src/github.com/hashicorp/raft/replication.go:135 +0x258
github.com/hashicorp/raft.(*Raft).startStopReplication.func1()
 /workspace/src/github.com/hashicorp/raft/raft.go:451 +0x33
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc4204c6b00, 0xc42045caa0)
 /workspace/src/github.com/hashicorp/raft/state.go:146 +0x53
created by github.com/hashicorp/raft.(*raftState).goFunc
 /workspace/src/github.com/hashicorp/raft/state.go:144 +0x66

goroutine 219 [chan receive, 24 minutes]:
gopkg.in/check%2ev1.(*suiteRunner).runFunc(0xc420298880, 0xc4201bb7a0, 0x0, 0xc4206395f0, 0x26, 0x0, 0xba71e0, 0xc4203f7ea8)
 /workspace/src/gopkg.in/check.v1/check.go:674 +0x91
gopkg.in/check%2ev1.(*suiteRunner).runFixture(0xc420298880, 0xc4201bb7a0, 0xc4206395f0, 0x26, 0x0, 0xc4203ce1a8)
 /workspace/src/gopkg.in/check.v1/check.go:717 +0x71
gopkg.in/check%2ev1.(*suiteRunner).runFixtureWithPanic(0xc420298880, 0xc4201bb7a0, 0xc4206395f0, 0x26, 0x0, 0xc4200aaf07, 0xdc6e170a25c39dd)
 /workspace/src/gopkg.in/check.v1/check.go:735 +0x6c
gopkg.in/check%2ev1.(*suiteRunner).forkTest.func1(0xc4201d8870)
 /workspace/src/gopkg.in/check.v1/check.go:773 +0x576
gopkg.in/check%2ev1.(*suiteRunner).forkCall.func1(0xc420298880, 0xc4201d8870, 0xc4206045e0)
 /workspace/src/gopkg.in/check.v1/check.go:666 +0x7c
created by gopkg.in/check%2ev1.(*suiteRunner).forkCall
 /workspace/src/gopkg.in/check.v1/check.go:663 +0x20c

goroutine 199 [select, 24 minutes]:
github.com/juju/pubsub.(*subscriber).loop(0xc4203854a0)
 /workspace/src/github.com/juju/pubsub/subscriber.go:59 +0x10d
created by github.com/juju/pubsub.newSubscriber
 /workspace/src/github.com/juju/pubsub/subscriber.go:40 +0x14d

goroutine 201 [select, 24 minutes]:
github.com/juju/juju/worker/catacomb.Invoke.func2(0xc4201ba690)
 /workspace/src/github.com/juju/juju/worker/catacomb/catacomb.go:101 +0x14d
created by github.com/juju/juju/worker/catacomb.Invoke
 /workspace/src/github.com/juju/juju/worker/catacomb/catacomb.go:98 +0x1c0

goroutine 253 [chan send, 24 minutes]:
github.com/hashicorp/raft.(*InmemTransport).makeRPC(0xc4206196e0, 0xc42060d800, 0x24, 0xa84f00, 0xc420523e00, 0x0, 0x0, 0x2faf080, 0x5afb7899, 0xc420567d00, ...)
 /workspace/src/github.com/hashicorp/raft/inmem_transport.go:143 +0x15d
github.com/hashicorp/raft.(*InmemTransport).AppendEntries(0xc4206196e0, 0xc420210e30, 0x9, 0xc42060d800, 0x24, 0xc420523e00, 0xc420293ba0, 0xbfae20, 0xc4203f3560)
 /workspace/src/github.com/hashicorp/raft/inmem_transport.go:94 +0x76
github.com/hashicorp/raft.(*Raft).heartbeat(0xc4204c6b00, 0xc4202b8180, 0xc4202274a0)
 /workspace/src/github.com/hashicorp/raft/replication.go:348 +0x347
github.com/hashicorp/raft.(*Raft).replicate.func1()
 /workspace/src/github.com/hashicorp/raft/replication.go:118 +0x3c
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc4204c6b00, 0xc420533e00)
 /workspace/src/github.com/hashicorp/raft/state.go:146 +0x53
created by github.com/hashicorp/raft.(*raftState).goFunc
 /workspace/src/github.com/hashicorp/raft/state.go:144 +0x66
FAIL github.com/juju/juju/worker/raft/raftclusterer 1501.707s

Revision history for this message
Christian Muirhead (2-xtian) wrote :
Revision history for this message
Heather Lanigan (hmlanigan) wrote :

Should this be fix committed? Or even fix released?

Revision history for this message
Anastasia (anastasia-macmood) wrote :

Marking as Fix Released as the PR landed a year ago.

Changed in juju:
importance: Undecided → Medium
status: Triaged → Fix Released
To post a comment you must log in.
This report contains Public information  
Everyone can see this information.

Other bug subscribers

Remote bug watches

Bug watches keep track of this bug in other bug trackers.