snapd: panic & runtime error on CentOS

Bug #1821186 reported by Pavel Mir
6
This bug affects 1 person
Affects Status Importance Assigned to Milestone
snapd
Fix Released
Undecided
Maciej Borzecki

Bug Description

A CentOS 7.6 instance with latests updates does not work with snapd,
it crashes on startup.

# uname -a
Linux linux-builder-02 3.10.0-957.10.1.el7.x86_64 #1 SMP Mon Mar 18 15:06:45 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux

# cat /proc/meminfo
MemTotal: 32742684 kB
MemFree: 25363796 kB
MemAvailable: 30603012 kB

# cat /etc/*release*
CentOS Linux release 7.6.1810 (Core)
Derived from Red Hat Enterprise Linux 7.6 (Source)
NAME="CentOS Linux"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"
CentOS Linux release 7.6.1810 (Core)

# yum info snapd
Loaded plugins: copr, fastestmirror
Loading mirror speeds from cached hostfile
 * base: mirror.corbina.net
 * epel: fedora-mirror01.rbc.ru
 * extras: centos-mirror.rbc.ru
 * ius: ftp.acc.umu.se
 * updates: centos-mirror.rbc.ru
Installed Packages
Name : snapd
Arch : x86_64
Version : 2.37.4
Release : 2.el7
Size : 41 M
Repo : installed
From repo : epel
Summary : A transactional software package manager
URL : https://github.com/snapcore/snapd
License : GPLv3
Description : Snappy is a modern, cross-distribution, transactional package manager
            : designed for working with self-contained, immutable packages.

# SNAPD_DEBUG=1 /usr/libexec/snapd/snapd
2019/03/21 16:37:44.016183 daemon.go:379: started snapd/2.37.4-2.el7 (series 16; classic; devmode) centos/7 (amd64) linux/3.10.0-957.10.1.el7.x86_6.
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0xa43cbf]

goroutine 22 [running]:
panic(0xb04da0, 0x138e5c0)
        /usr/lib/golang/src/runtime/panic.go:556 +0x2cb fp=0xc00008dda8 sp=0xc00008dd18 pc=0x42c6fb
runtime.panicmem()
        /usr/lib/golang/src/runtime/panic.go:82 +0x5e fp=0xc00008ddc8 sp=0xc00008dda8 pc=0x42b73e
runtime.sigpanic()
        /usr/lib/golang/src/runtime/signal_unix.go:390 +0x182 fp=0xc00008de18 sp=0xc00008ddc8 pc=0x441d12
github.com/snapcore/snapd/daemon.(*shutdownServer).CanStandby(0x0, 0x0)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/daemon/daemon.go:444 +0x3f fp=0xc00008dea8 sp=0xc00008de18 pc=0xa43cbf
github.com/snapcore/snapd/overlord/standby.(*StandbyOpinions).CanStandby(0xc0002fde50, 0x2540be400)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/overlord/standby/standby.go:67 +0x212 fp=0xc00008df38 sp=0xc00008dea8 pc=0xa1f972
github.com/snapcore/snapd/overlord/standby.(*StandbyOpinions).Start.func1(0xc0002fde50)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/overlord/standby/standby.go:88 +0xa1 fp=0xc00008dfd8 sp=0xc00008df38 pc=0xa1feb1
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00008dfe0 sp=0xc00008dfd8 pc=0x459f21
created by github.com/snapcore/snapd/overlord/standby.(*StandbyOpinions).Start
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/overlord/standby/standby.go:84 +0x3f

goroutine 1 [select]:
runtime.gopark(0xc5f598, 0x0, 0x1809, 0x1)
        /usr/lib/golang/src/runtime/proc.go:302 +0xeb fp=0xc000165c10 sp=0xc000165bf0 pc=0x42e81b
runtime.selectgo(0xc000165e58, 0xc000165dc4, 0x3, 0x1, 0x1)
        /usr/lib/golang/src/runtime/select.go:313 +0xcc6 fp=0xc000165d70 sp=0xc000165c10 pc=0x43e006
main.run(0xc000222a80, 0x0, 0x0)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/cmd/snapd/main.go:145 +0x4a4 fp=0xc000165ee0 sp=0xc000165d70 pc=0xa53014
main.main()
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/cmd/snapd/main.go:58 +0xd5 fp=0xc000165f98 sp=0xc000165ee0 pc=0xa52765
runtime.main()
        /usr/lib/golang/src/runtime/proc.go:201 +0x207 fp=0xc000165fe0 sp=0xc000165f98 pc=0x42e427
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc000165fe8 sp=0xc000165fe0 pc=0x459f21

goroutine 2 [force gc (idle)]:
runtime.gopark(0xc5f558, 0x13d45f0, 0x1410, 0x1)
        /usr/lib/golang/src/runtime/proc.go:302 +0xeb fp=0xc00007ef80 sp=0xc00007ef60 pc=0x42e81b
runtime.goparkunlock(0x13d45f0, 0x1410, 0x1)
        /usr/lib/golang/src/runtime/proc.go:308 +0x53 fp=0xc00007efb0 sp=0xc00007ef80 pc=0x42e8c3
runtime.forcegchelper()
        /usr/lib/golang/src/runtime/proc.go:251 +0xb3 fp=0xc00007efe0 sp=0xc00007efb0 pc=0x42e693
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00007efe8 sp=0xc00007efe0 pc=0x459f21
created by runtime.init.5
        /usr/lib/golang/src/runtime/proc.go:240 +0x35

goroutine 3 [GC sweep wait]:
runtime.gopark(0xc5f558, 0x13d4760, 0x42140c, 0x1)
        /usr/lib/golang/src/runtime/proc.go:302 +0xeb fp=0xc00007f780 sp=0xc00007f760 pc=0x42e81b
runtime.goparkunlock(0x13d4760, 0xcf140c, 0x1)
        /usr/lib/golang/src/runtime/proc.go:308 +0x53 fp=0xc00007f7b0 sp=0xc00007f780 pc=0x42e8c3
runtime.bgsweep(0xc0000a8000)
        /usr/lib/golang/src/runtime/mgcsweep.go:52 +0x8f fp=0xc00007f7d8 sp=0xc00007f7b0 pc=0x42185f
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00007f7e0 sp=0xc00007f7d8 pc=0x459f21
created by runtime.gcenable
        /usr/lib/golang/src/runtime/mgc.go:216 +0x58

goroutine 4 [finalizer wait]:
runtime.gopark(0xc5f558, 0x13f2c90, 0x140f, 0x1)
        /usr/lib/golang/src/runtime/proc.go:302 +0xeb fp=0xc00007ff28 sp=0xc00007ff08 pc=0x42e81b
runtime.goparkunlock(0x13f2c90, 0x140f, 0x1)
        /usr/lib/golang/src/runtime/proc.go:308 +0x53 fp=0xc00007ff58 sp=0xc00007ff28 pc=0x42e8c3
runtime.runfinq()
        /usr/lib/golang/src/runtime/mfinal.go:175 +0x99 fp=0xc00007ffe0 sp=0xc00007ff58 pc=0x419219
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00007ffe8 sp=0xc00007ffe0 pc=0x459f21
created by runtime.createfing
        /usr/lib/golang/src/runtime/mfinal.go:156 +0x61

goroutine 5 [syscall]:
runtime.notetsleepg(0x13f3100, 0xffffffffffffffff, 0x0)
        /usr/lib/golang/src/runtime/lock_futex.go:227 +0x37 fp=0xc000080788 sp=0xc000080758 pc=0x40c877
os/signal.signal_recv(0x0)
        /usr/lib/golang/src/runtime/sigqueue.go:139 +0x9c fp=0xc0000807b0 sp=0xc000080788 pc=0x442c9c
os/signal.loop()
        /usr/lib/golang/src/os/signal/signal_unix.go:23 +0x22 fp=0xc0000807e0 sp=0xc0000807b0 pc=0x4dc012
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000807e8 sp=0xc0000807e0 pc=0x459f21
created by os/signal.init.0
        /usr/lib/golang/src/os/signal/signal_unix.go:29 +0x41

goroutine 6 [select, locked to thread]:
runtime.gopark(0xc5f598, 0x0, 0x1809, 0x1)
        /usr/lib/golang/src/runtime/proc.go:302 +0xeb fp=0xc00007e5e0 sp=0xc00007e5c0 pc=0x42e81b
runtime.selectgo(0xc00007e780, 0xc00007e778, 0x2, 0x0, 0xc000094101)
        /usr/lib/golang/src/runtime/select.go:313 +0xcc6 fp=0xc00007e740 sp=0xc00007e5e0 pc=0x43e006
runtime.ensureSigM.func1()
        /usr/lib/golang/src/runtime/signal_unix.go:547 +0x1bf fp=0xc00007e7e0 sp=0xc00007e740 pc=0x45767f
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00007e7e8 sp=0xc00007e7e0 pc=0x459f21
created by runtime.ensureSigM
        /usr/lib/golang/src/runtime/signal_unix.go:530 +0xd5

goroutine 34 [timer goroutine (idle)]:
runtime.gopark(0xc5f558, 0x13d9480, 0x12c4b1414, 0x1)
        /usr/lib/golang/src/runtime/proc.go:302 +0xeb fp=0xc00007bf28 sp=0xc00007bf08 pc=0x42e81b
runtime.goparkunlock(0x13d9480, 0xc000301414, 0x1)
        /usr/lib/golang/src/runtime/proc.go:308 +0x53 fp=0xc00007bf58 sp=0xc00007bf28 pc=0x42e8c3
runtime.timerproc(0x13d9480)
        /usr/lib/golang/src/runtime/time.go:280 +0x288 fp=0xc00007bfd8 sp=0xc00007bf58 pc=0x44b138
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00007bfe0 sp=0xc00007bfd8 pc=0x459f21
created by runtime.(*timersBucket).addtimerLocked
        /usr/lib/golang/src/runtime/time.go:170 +0x114

goroutine 23 [syscall]:
runtime.notetsleepg(0x13d9420, 0x2540ae821, 0x1)
        /usr/lib/golang/src/runtime/lock_futex.go:227 +0x37 fp=0xc00007cf58 sp=0xc00007cf28 pc=0x40c877
runtime.timerproc(0x13d9400)
        /usr/lib/golang/src/runtime/time.go:288 +0x30e fp=0xc00007cfd8 sp=0xc00007cf58 pc=0x44b1be
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00007cfe0 sp=0xc00007cfd8 pc=0x459f21
created by runtime.(*timersBucket).addtimerLocked
        /usr/lib/golang/src/runtime/time.go:170 +0x114

goroutine 24 [select]:
runtime.gopark(0xc5f598, 0x0, 0x1809, 0x1)
        /usr/lib/golang/src/runtime/proc.go:302 +0xeb fp=0xc0000ebd68 sp=0xc0000ebd48 pc=0x42e81b
runtime.selectgo(0xc0000ebf10, 0xc0000ebef4, 0x3, 0x13d4ee0, 0x0)
        /usr/lib/golang/src/runtime/select.go:313 +0xcc6 fp=0xc0000ebec8 sp=0xc0000ebd68 pc=0x43e006
github.com/snapcore/snapd/overlord.(*Overlord).Loop.func1(0x0, 0x0)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/overlord/overlord.go:281 +0x10f fp=0xc0000ebf98 sp=0xc0000ebec8 pc=0xa1f38f
github.com/snapcore/snapd/vendor/gopkg.in/tomb%2ev2.(*Tomb).run(0xc0002fc000, 0xc0002f1800)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/vendor/gopkg.in/tomb.v2/tomb.go:163 +0x2b fp=0xc0000ebfd0 sp=0xc0000ebf98 pc=0x647b7b
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000ebfd8 sp=0xc0000ebfd0 pc=0x459f21
created by github.com/snapcore/snapd/vendor/gopkg.in/tomb%2ev2.(*Tomb).Go
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/vendor/gopkg.in/tomb.v2/tomb.go:159 +0xba

goroutine 25 [IO wait]:
runtime.gopark(0xc5f528, 0x7f682ffd4f28, 0x1b02, 0x5)
        /usr/lib/golang/src/runtime/proc.go:302 +0xeb fp=0xc00008cb20 sp=0xc00008cb00 pc=0x42e81b
runtime.netpollblock(0x7f682ffd4f00, 0x72, 0xc0003f6000)
        /usr/lib/golang/src/runtime/netpoll.go:366 +0x99 fp=0xc00008cb58 sp=0xc00008cb20 pc=0x429b49
internal/poll.runtime_pollWait(0x7f682ffd4f00, 0x72, 0x0)
        /usr/lib/golang/src/runtime/netpoll.go:173 +0x66 fp=0xc00008cb88 sp=0xc00008cb58 pc=0x4291f6
internal/poll.(*pollDesc).wait(0xc000309918, 0x72, 0xc00030e000, 0x0, 0x0)
        /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:85 +0x9a fp=0xc00008cbb8 sp=0xc00008cb88 pc=0x494a9a
internal/poll.(*pollDesc).waitRead(0xc000309918, 0xffffffffffffff00, 0x0, 0x0)
        /usr/lib/golang/src/internal/poll/fd_poll_runtime.go:90 +0x3d fp=0xc00008cbf0 sp=0xc00008cbb8 pc=0x494bad
internal/poll.(*FD).Accept(0xc000309900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/lib/golang/src/internal/poll/fd_unix.go:384 +0x1a0 fp=0xc00008cc58 sp=0xc00008cbf0 pc=0x496eb0
net.(*netFD).accept(0xc000309900, 0xc0003ad2f0, 0x30, 0x50)
        /usr/lib/golang/src/net/fd_unix.go:238 +0x42 fp=0xc00008cd28 sp=0xc00008cc58 pc=0x689732
net.(*UnixListener).accept(0xc00037d7a0, 0x30, 0xb72640, 0x0)
        /usr/lib/golang/src/net/unixsock_posix.go:162 +0x32 fp=0xc00008cd60 sp=0xc00008cd28 pc=0x6adf32
net.(*UnixListener).Accept(0xc00037d7a0, 0xc0002dc000, 0x50, 0x7f68300206c0, 0x0)
        /usr/lib/golang/src/net/unixsock.go:257 +0x47 fp=0xc00008cda0 sp=0xc00008cd60 pc=0x6abe37
github.com/snapcore/snapd/daemon.(*ucrednetListener).Accept(0xc0002f0d50, 0x0, 0x0, 0x0, 0x0)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/daemon/ucrednet.go:105 +0x4d fp=0xc00008ce20 sp=0xc00008cda0 pc=0xa4c6ad
net/http.(*onceCloseListener).Accept(0xc0003ad260, 0xc000032390, 0xaf8fa0, 0x138e4e0, 0xbb4ea0)
        <autogenerated>:1 +0x3c fp=0xc00008ce58 sp=0xc00008ce20 pc=0x7dfa9c
net/http.(*Server).Serve(0xc0003f4000, 0xd07840, 0xc0002f0d50, 0x0, 0x0)
        /usr/lib/golang/src/net/http/server.go:2826 +0x22f fp=0xc00008cf18 sp=0xc00008ce58 pc=0x7bda5f
github.com/snapcore/snapd/daemon.(*shutdownServer).Serve(0xc0003ad230, 0x0, 0x0)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/daemon/daemon.go:440 +0x40 fp=0xc00008cf50 sp=0xc00008cf18 pc=0xa43c50
github.com/snapcore/snapd/daemon.(*Daemon).Start.func2(0x0, 0x0)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/daemon/daemon.go:562 +0x42 fp=0xc00008cf98 sp=0xc00008cf50 pc=0xa4d312
github.com/snapcore/snapd/vendor/gopkg.in/tomb%2ev2.(*Tomb).run(0xc0002f2108, 0xc0002f1810)
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/vendor/gopkg.in/tomb.v2/tomb.go:163 +0x2b fp=0xc00008cfd0 sp=0xc00008cf98 pc=0x647b7b
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00008cfd8 sp=0xc00008cfd0 pc=0x459f21
created by github.com/snapcore/snapd/vendor/gopkg.in/tomb%2ev2.(*Tomb).Go
        /builddir/build/BUILD/snapd-2.37.4/src/github.com/snapcore/snapd/vendor/gopkg.in/tomb.v2/tomb.go:159 +0xba
Aborted

Revision history for this message
Pavel Mir (previewthenew) wrote :

I've downgraded kernel to 3.10.0-957.5.1 and all works fine

# snap version
snap 2.37.4-2.el7
snapd 2.37.4-2.el7
series 16
centos 7
kernel 3.10.0-957.5.1.el7.x86_64

# uname -a
Linux linux-builder-02 3.10.0-957.5.1.el7.x86_64 #1 SMP Fri Feb 1 14:54:57 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux

Revision history for this message
Zygmunt Krynicki (zyga) wrote :

We discussed this issue among the snapd team and we believe that this is a race caused by incorrect initialization of signal handlers. This bug was fixed a while ago. I've assigned this to my colleague to confirm.

Changed in snapd:
status: New → Fix Released
assignee: nobody → Maciej Borzecki (maciek-borzecki)
To post a comment you must log in.
This report contains Public information  
Everyone can see this information.

Other bug subscribers

Remote bug watches

Bug watches keep track of this bug in other bug trackers.