I'm trying to run with this patch to figure out why we're creating these managers: diff --git a/worker/lease/manager.go b/worker/lease/manager.go index b5de658..9e18ba0 100644 --- a/worker/lease/manager.go +++ b/worker/lease/manager.go @@ -4,6 +4,7 @@ package lease import ( + "runtime/debug" "sort" "time" @@ -63,6 +64,7 @@ func NewManager(config ManagerConfig) (*Manager, error) { if err != nil { return nil, errors.Trace(err) } + logger.Debugf("creating manager %p for:\n%s", manager, string(debug.Stack())) return manager, nil } @@ -232,12 +234,13 @@ func (manager *Manager) nextTick() <-chan time.Time { now := manager.config.Clock.Now() nextTick := now.Add(manager.config.MaxSleep) for _, info := range manager.config.Client.Leases() { + leases := manager.config.Client.Leases() if info.Expiry.After(nextTick) { continue } nextTick = info.Expiry } - logger.Debugf("waking to check leases at %s", nextTick) + logger.Debugf("[%p] waking to check (%d) leases at %s", manager, len(leases), nextTick) return clock.Alarm(manager.config.Clock, nextTick) } I ended up needing to rebootstrap because I upgraded the source tree in the mean time. But logging again and trimming it up a bit I see: 19:30:15.372 ... [128460] wak... (1) 15:31:15.3720 19:30:19.933 ... [2a0960] wak... (0) 15:31:19.9339 19:30:20.652 ... [3de3c0] wak... (0) 15:31:20.6528 19:30:36.085 ... [1881e0] wak... (1) 15:31:36.0857 19:30:36.094 ... [1281e0] wak... (2) 15:31:06.0859 19:30:36.578 ... [1268c0] wak... (2) 15:31:06.5696 19:30:36.670 ... [1281e0] wak... (2) 15:31:36.6708 19:30:45.377 ... [128460] wak... (1) 15:31:45.3776 19:31:06.139 ... [1281e0] wak... (2) 15:31:37.1316 19:31:06.145 ... [1881e0] wak... (1) 15:32:06.1456 19:31:06.573 ... [1268c0] wak... (2) 15:31:36.5645 19:31:06.712 ... [1281e0] wak... (2) 15:32:05.7122 19:31:15.380 ... [128460] wak... (1) 15:32:15.3808 19:31:19.936 ... [2a0960] wak... (0) 15:32:19.9369 19:31:20.656 ... [3de3c0] wak... (0) 15:32:20.6560 19:31:36.194 ... [1281e0] wak... (2) 15:32:07.1841 19:31:36.198 ... [1881e0] wak... (1) 15:32:36.1980 19:31:36.567 ... [1268c0] wak... (2) 15:32:06.5577 19:31:36.754 ... [1281e0] wak... (2) 15:32:35.7545 19:31:45.381 ... [128460] wak... (1) 15:32:45.3811 19:32:06.244 ... [1281e0] wak... (2) 15:32:37.2340 19:32:06.250 ... [1881e0] wak... (1) 15:33:06.2503 19:32:06.561 ... [1268c0] wak... (2) 15:32:37.5508 19:32:06.792 ... [1281e0] wak... (2) 15:33:06.7926 19:32:15.381 ... [128460] wak... (1) 15:33:15.3812 19:32:19.940 ... [2a0960] wak... (0) 15:33:19.9400 19:32:20.693 ... [3de3c0] wak... (0) 15:33:20.6931 19:32:36.298 ... [1281e0] wak... (2) 15:33:06.2892 19:32:36.303 ... [1881e0] wak... (1) 15:33:36.3031 19:32:36.835 ... [1281e0] wak... (2) 15:33:36.8352 19:32:37.554 ... [1268c0] wak... (2) 15:33:36.5540 19:32:45.375 ... [128460] wak... (1) 15:33:45.3756 19:33:06.347 ... [1281e0] wak... (2) 15:33:37.3377 19:33:06.350 ... [1881e0] wak... (1) 15:34:06.3503 19:33:06.876 ... [1281e0] wak... (2) 15:34:05.8762 19:33:15.379 ... [128460] wak... (1) 15:34:15.3791 19:33:19.943 ... [2a0960] wak... (0) 15:34:19.9432 19:33:20.696 ... [3de3c0] wak... (0) 15:34:20.6964 19:33:36.397 ... [1281e0] wak... (2) 15:34:07.3871 19:33:36.400 ... [1881e0] wak... (1) 15:34:36.4001 19:33:36.557 ... [1268c0] wak... (2) 15:34:07.5470 19:33:36.911 ... [1281e0] wak... (2) 15:34:35.9114 19:33:45.378 ... [128460] wak... (1) 15:34:45.3781 19:34:06.453 ... [1281e0] wak... (2) 15:34:37.4451 19:34:06.456 ... [1881e0] wak... (1) 15:35:06.4560 19:34:06.951 ... [1281e0] wak... (2) 15:35:06.9512 19:34:07.549 ... [1268c0] wak... (2) 15:35:06.5492 19:34:15.369 ... [128460] wak... (1) 15:35:15.3698 19:34:19.945 ... [2a0960] wak... (0) 15:35:19.9453 19:34:20.699 ... [3de3c0] wak... (0) 15:35:20.6997 19:34:36.506 ... [1281e0] wak... (2) 15:35:06.4979 19:34:36.511 ... [1881e0] wak... (1) 15:35:36.5110 19:34:36.989 ... [1281e0] wak... (2) 15:35:36.9891 19:34:45.379 ... [128460] wak... (1) 15:35:45.3798 By just doing a sort | uniq -c of just the Manager memory addresses, I can see: 7 1268c0 18 1281e0 10 128460 9 1881e0 5 2a0960 5 3de3c0 so there are 2 managers that are not managing *any* leases. 1 that is managing 2 leases, but only waking up 7 times. Another one that is managing 2 leases but woke up 18 times. One that is managing only a single lease but woke up 10 times (presumably that one is the application lease that should be getting refreshed every 30s). its still rather surprising to see to managers running that aren't actually managing anything.