go panic

Bug #1626381 reported by Kyle Nitzsche
6
This bug affects 1 person
Affects Status Importance Assigned to Milestone
Yelp Scope
High
Zhang Enwei

Bug Description

com.canonical.scopes.yelp 1.6

goroutine 21 [running]:
runtime.throw(0x463feb)
 /usr/lib/go/src/pkg/runtime/panic.c:520 +0x5c fp=0xd4510bfc sp=0xd4510bf0
runtime.MCentral_CacheSpan(0x48944c)
 /usr/lib/go/src/pkg/runtime/mcentral.c:96 +0x124 fp=0xd4510c1c sp=0xd4510bfc
runtime.MCache_Refill(0xc4332000, 0x2)
 /usr/lib/go/src/pkg/runtime/mcache.c:78 +0x110 fp=0xd4510c34 sp=0xd4510c1c
runtime.mallocgc(0x10, 0x2cfb38, 0x0)
 /usr/lib/go/src/pkg/runtime/malloc.goc:152 +0x2d8 fp=0xd4510c6c sp=0xd4510c34
runtime.new(0x2cfb38, 0x0)
 /usr/lib/go/src/pkg/runtime/malloc.goc:826 +0x34 fp=0xd4510c7c sp=0xd4510c6c
reflect.MakeSlice(0xc4342118, 0x27a250, 0x0, 0x4, 0x0, 0x0, 0x0, 0x0)
 /usr/lib/go/src/pkg/reflect/value.go:2228 +0x1bc fp=0xd4510cc8 sp=0xd4510c7c
encoding/json.(*decodeState).array(0xd44aa014, 0x27a250, 0xd4485114, 0x0, 0x176)
 /usr/lib/go/src/pkg/encoding/json/decode.go:403 +0x584 fp=0xd4510dd8 sp=0xd4510cc8
encoding/json.(*decodeState).value(0xd44aa014, 0x27a250, 0xd4485114, 0x0, 0x176)
 /usr/lib/go/src/pkg/encoding/json/decode.go:286 +0x350 fp=0xd4510e54 sp=0xd4510dd8
encoding/json.(*decodeState).object(0xd44aa014, 0x2e4908, 0xd44850ac, 0x0, 0x196)
 /usr/lib/go/src/pkg/encoding/json/decode.go:499 +0x9b8 fp=0xd4510ff8 sp=0xd4510e54
encoding/json.(*decodeState).value(0xd44aa014, 0x2e4908, 0xd44850ac, 0x0, 0x196)
 /usr/lib/go/src/pkg/encoding/json/decode.go:289 +0x324 fp=0xd4511074 sp=0xd4510ff8
encoding/json.(*decodeState).array(0xd44aa014, 0x279c38, 0xd45a2620, 0x0, 0x176)
 /usr/lib/go/src/pkg/encoding/json/decode.go:414 +0x668 fp=0xd4511184 sp=0xd4511074
encoding/json.(*decodeState).value(0xd44aa014, 0x279c38, 0xd45a2620, 0x0, 0x176)
 /usr/lib/go/src/pkg/encoding/json/decode.go:286 +0x350 fp=0xd4511200 sp=0xd4511184
encoding/json.(*decodeState).object(0xd44aa014, 0x2c8e28, 0xd45a2620, 0x0, 0x196)
 /usr/lib/go/src/pkg/encoding/json/decode.go:499 +0x9b8 fp=0xd45113a4 sp=0xd4511200
encoding/json.(*decodeState).value(0xd44aa014, 0x275e80, 0xd45a2620, 0x0, 0x160)
 /usr/lib/go/src/pkg/encoding/json/decode.go:289 +0x324 fp=0xd4511420 sp=0xd45113a4
encoding/json.(*decodeState).unmarshal(0xd44aa014, 0x275e80, 0xd45a2620, 0x0, 0x0)
 /usr/lib/go/src/pkg/encoding/json/decode.go:149 +0x184 fp=0xd451148c sp=0xd4511420
encoding/json.(*Decoder).Decode(0xd44aa000, 0x275e80, 0xd45a2620, 0x0, 0x0)
 /usr/lib/go/src/pkg/encoding/json/stream.go:53 +0x148 fp=0xd45114ec sp=0xd451148c
main.(*YelpScope).get(0xd44a0300, 0xd45a2640, 0x275e80, 0xd45a2620, 0x0, 0x0)
 /home/enwei/work/20151225/upload/yelp-scope/src/yelp.go:192 +0x250 fp=0xd4511554 sp=0xd45114ec
main.(*YelpScope).Search(0xd44a0300, 0xd44a68f0, 0xd44a6900, 0xd454e290, 0xd44a0000, 0x0, 0x0)
 /home/enwei/work/20151225/upload/yelp-scope/src/yelp.go:316 +0x2400 fp=0xd4511f98 sp=0xd4511554
launchpad.net/go-unityscopes/v2.func·002()
 /home/enwei/go/src/launchpad.net/go-unityscopes/v2/unityscope.go:70 +0x80 fp=0xd4511fcc sp=0xd4511f98
runtime.goexit()
 /usr/lib/go/src/pkg/runtime/proc.c:1445 fp=0xd4511fcc sp=0xd4511fcc
created by launchpad.net/go-unityscopes/v2.callScopeSearch
 /home/enwei/go/src/launchpad.net/go-unityscopes/v2/unityscope.go:76 +0x150

goroutine 16 [syscall]:
launchpad.net/go-unityscopes/v2._Cfunc_run_scope(0xd44a63b0, 0xd44a60f0, 0xd44a6108, 0xd44a63a8, 0xd44a63b8)
 launchpad.net/go-unityscopes/v2/_obj/_cgo_defun.c:667 +0x34
launchpad.net/go-unityscopes/v2.Run(0xc43424a8, 0xd44a0300, 0x0, 0x0)
 /home/enwei/go/src/launchpad.net/go-unityscopes/v2/unityscope.go:257 +0x31c
main.main()
 /home/enwei/work/20151225/upload/yelp-scope/src/yelp.go:693 +0x1d0

goroutine 19 [finalizer wait]:
runtime.park(0x438fc, 0x479a2c, 0x467e51)
 /usr/lib/go/src/pkg/runtime/proc.c:1369 +0x5c
runtime.parkunlock(0x479a2c, 0x467e51)
 /usr/lib/go/src/pkg/runtime/proc.c:1385 +0x40
runfinq()
 /usr/lib/go/src/pkg/runtime/mgc0.c:2644 +0xa0
runtime.goexit()
 /usr/lib/go/src/pkg/runtime/proc.c:1445

goroutine 17 [syscall]:
runtime.goexit()
 /usr/lib/go/src/pkg/runtime/proc.c:1445

goroutine 18 [syscall]:
runtime.goexit()
 /usr/lib/go/src/pkg/runtime/proc.c:1445

goroutine 20 [select]:
net/http.(*persistConn).roundTrip(0xd4476120, 0xd45a4140, 0x0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/transport.go:1015 +0x634
net/http.(*Transport).RoundTrip(0xd4476240, 0xd44721c0, 0xf9ab0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/transport.go:208 +0x388
net/http.send(0xd44721c0, 0xc43423d0, 0xd4476240, 0xd44724d0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/client.go:195 +0x378
net/http.(*Client).send(0xd45a2080, 0xd44721c0, 0x6d, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/client.go:118 +0x140
net/http.(*Client).doFollowingRedirects(0xd45a2080, 0xd44721c0, 0x377308, 0x0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/client.go:343 +0x808
net/http.(*Client).Do(0xd45a2080, 0xd44721c0, 0xd4472070, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/client.go:150 +0xb4
main.(*YelpScope).get(0xd44a0300, 0xd45a2020, 0x275e80, 0xd45a2000, 0x0, 0x0)
 /home/enwei/work/20151225/upload/yelp-scope/src/yelp.go:189 +0x144
main.(*YelpScope).Search(0xd44a0300, 0xd44a63e0, 0xd44a63f0, 0xd44ac4e0, 0xd44a03c0, 0x0, 0x0)
 /home/enwei/work/20151225/upload/yelp-scope/src/yelp.go:316 +0x2400
launchpad.net/go-unityscopes/v2.func·002()
 /home/enwei/go/src/launchpad.net/go-unityscopes/v2/unityscope.go:70 +0x80
created by launchpad.net/go-unityscopes/v2.callScopeSearch
 /home/enwei/go/src/launchpad.net/go-unityscopes/v2/unityscope.go:76 +0x150

goroutine 29 [IO wait]:
net.runtime_pollWait(0xc434b6f0, 0x72, 0x0)
 /usr/lib/go/src/pkg/runtime/netpoll.goc:146 +0x6c
net.(*pollDesc).Wait(0xd44a03b8, 0x72, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/fd_poll_runtime.go:84 +0x44
net.(*pollDesc).WaitRead(0xd44a03b8, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/fd_poll_runtime.go:89 +0x40
net.(*netFD).Read(0xd44a0380, 0xd449f000, 0x1000, 0x1000, 0x0, 0xc43422c8, 0xb)
 /usr/lib/go/src/pkg/net/fd_unix.go:242 +0x320
net.(*conn).Read(0xd45a4060, 0xd449f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/net.go:122 +0xd4
net/http.noteEOFReader.Read(0xc434b760, 0xd45a4060, 0xd447614c, 0xd449f000, 0x1000, 0x1000, 0x8, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/transport.go:1203 +0x80
net/http.(*noteEOFReader).Read(0xd45a62d0, 0xd449f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
 <autogenerated>:124 +0xdc
bufio.(*Reader).fill(0xd44804e0)
 /usr/lib/go/src/pkg/bufio/bufio.go:97 +0x1cc
bufio.(*Reader).Peek(0xd44804e0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
 /usr/lib/go/src/pkg/bufio/bufio.go:132 +0xdc
net/http.(*persistConn).readLoop(0xd4476120)
 /usr/lib/go/src/pkg/net/http/transport.go:782 +0x90
created by net/http.(*Transport).dialConn
 /usr/lib/go/src/pkg/net/http/transport.go:600 +0x85c

goroutine 28 [select]:
net/http.(*persistConn).writeLoop(0xd44760c0)
 /usr/lib/go/src/pkg/net/http/transport.go:885 +0x2d4
created by net/http.(*Transport).dialConn
 /usr/lib/go/src/pkg/net/http/transport.go:601 +0x884

goroutine 27 [select]:
net/http.(*persistConn).readLoop(0xd44760c0)
 /usr/lib/go/src/pkg/net/http/transport.go:868 +0x74c
created by net/http.(*Transport).dialConn
 /usr/lib/go/src/pkg/net/http/transport.go:600 +0x85c

goroutine 30 [select]:
net/http.(*persistConn).writeLoop(0xd4476120)
 /usr/lib/go/src/pkg/net/http/transport.go:885 +0x2d4
created by net/http.(*Transport).dialConn
 /usr/lib/go/src/pkg/net/http/transport.go:601 +0x884
[2016-09-22 00:24:11.355] INFO: Registry: RegistryObject::ScopeProcess::on_process_death(): Process for scope: "com.canonical.scopes.yelp_yelp" exited
[2016-09-22 00:24:11.355] ERROR: Registry: RegistryObject::ScopeProcess: Scope: "com.canonical.scopes.yelp_yelp" closed unexpectedly. Either the process crashed or was killed forcefully.
query complete, status: ok

Revision history for this message
Kyle Nitzsche (knitzsche) wrote :

Hi Enwei, this may be related to some changes you made (and maybe not ;).

Mind taking a look?

Changed in yelp-scope:
assignee: nobody → Zhang Enwei (zhangew401)
Revision history for this message
Zhang Enwei (zhangew401) wrote :

Sure, let me check.

Zhang Enwei (zhangew401)
Changed in yelp-scope:
status: New → In Progress
Revision history for this message
Zhang Enwei (zhangew401) wrote :

Hi Kyle,
Could you share your location to me?
I tried more than 20 times using faked location and it is not reproduced.
custom.location.lon=-122.399972
custom.location.lat=37.786882
Thanks.

Revision history for this message
Kyle Nitzsche (knitzsche) wrote :

Hi Enwei,

I *think* I saw the panic while using a fake location:
custom.prop is:

custom.location.fake=true
custom.location.lon=-0.0999525
custom.location.lat=51.5062348

(Which is London)

Revision history for this message
Zhang Enwei (zhangew401) wrote :

My first thought is to catch the panic, but I found it is throw by runtime.throw, not by runtime.panic. Therefore I cannot catch it by recover().

The "freelist empty" error generally means that the garbage collector has
collected a pointer that is still live. The GC put the memory on the
freelist, but then the memory was changed by the program, breaking the
freelist.

Changed in yelp-scope:
importance: Undecided → High
Revision history for this message
Zhang Enwei (zhangew401) wrote :
Download full text (7.3 KiB)

I met another panic while stress testing.
panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xb code=0x1 addr=0x0 pc=0x5e7b0]

goroutine 28 [running]:
runtime.panic(0x2d3a50, 0x46588b)
 /usr/lib/go/src/pkg/runtime/panic.c:279 +0x1a0
net/http.Header.sortedKeyValues(0x9442c5e0, 0x94324580, 0x0, 0x0, 0x2, 0x94430020)
 /usr/lib/go/src/pkg/net/http/header.go:135 +0x2e8
net/http.Header.WriteSubset(0x9442c5e0, 0x841d6860, 0x9442cc20, 0x94324580, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/header.go:150 +0xec
net/http.(*Request).write(0x943024d0, 0x841d6860, 0x9442cc20, 0x841d6800, 0x9442c700, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/request.go:420 +0x698
net/http.(*persistConn).writeLoop(0x94306c60)
 /usr/lib/go/src/pkg/net/http/transport.go:891 +0x1cc
created by net/http.(*Transport).dialConn
 /usr/lib/go/src/pkg/net/http/transport.go:601 +0x884

goroutine 16 [syscall]:
launchpad.net/go-unityscopes/v2._Cfunc_run_scope(0x943363b0, 0x943360f0, 0x94336108, 0x943363a8, 0x943363b8)
 launchpad.net/go-unityscopes/v2/_obj/_cgo_defun.c:667 +0x34
launchpad.net/go-unityscopes/v2.Run(0x841d14a8, 0x94330300, 0x0, 0x0)
 /home/enwei/go/src/launchpad.net/go-unityscopes/v2/unityscope.go:257 +0x31c
main.main()
 /home/enwei/scopes/yelp-scope/src/yelp.go:692 +0x1d0

goroutine 19 [finalizer wait]:
runtime.park(0x438fc, 0x479a2c, 0x467e51)
 /usr/lib/go/src/pkg/runtime/proc.c:1369 +0x5c
runtime.parkunlock(0x479a2c, 0x467e51)
 /usr/lib/go/src/pkg/runtime/proc.c:1385 +0x40
runfinq()
 /usr/lib/go/src/pkg/runtime/mgc0.c:2644 +0xa0
runtime.goexit()
 /usr/lib/go/src/pkg/runtime/proc.c:1445

goroutine 17 [syscall]:
runtime.goexit()
 /usr/lib/go/src/pkg/runtime/proc.c:1445

goroutine 18 [syscall]:
runtime.goexit()
 /usr/lib/go/src/pkg/runtime/proc.c:1445

goroutine 20 [select]:
net/http.(*persistConn).roundTrip(0x94306cc0, 0x9442f1b0, 0x0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/transport.go:1015 +0x634
net/http.(*Transport).RoundTrip(0x94306240, 0x943038f0, 0xf9ab0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/transport.go:208 +0x388
net/http.send(0x943038f0, 0x841d13d0, 0x94306240, 0x943039d0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/client.go:195 +0x378
net/http.(*Client).send(0x9442c880, 0x943038f0, 0x6d, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/client.go:118 +0x140
net/http.(*Client).doFollowingRedirects(0x9442c880, 0x943038f0, 0x3773a8, 0x0, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/client.go:343 +0x808
net/http.(*Client).Do(0x9442c880, 0x943038f0, 0x94303810, 0x0, 0x0)
 /usr/lib/go/src/pkg/net/http/client.go:150 +0xb4
main.(*YelpScope).get(0x94330300, 0x9442c840, 0x275e80, 0x9442c820, 0x0, 0x0)
 /home/enwei/scopes/yelp-scope/src/yelp.go:189 +0x144
main.(*YelpScope).Search(0x94330300, 0x943363e0, 0x943363f0, 0x9433c4d0, 0x94330400, 0x0, 0x0)
 /home/enwei/scopes/yelp-scope/src/yelp.go:315 +0x2400
launchpad.net/go-unityscopes/v2.func·002()
 /home/enwei/go/src/launchpad.net/go-unityscopes/v2/unityscope.go:70 +0x80
created by launchpad.net/go-unityscopes/v2.callScopeSearch
 /home/enwei/go/src/launchpad.net/go-unityscopes/v2/unityscope.go:76 +0x150

goroutine 21 [select]:
net/http.(*persistConn).roundTrip(0x94306c60, 0x9442e140, 0x0, 0x0, 0x0)
 /usr/lib/go/src/p...

Read more...

Revision history for this message
Zhang Enwei (zhangew401) wrote :

Hi Kyle,
Would you please help try this click?
I tried about 1000 times by using scope-data, it seems to be fixed.
Thanks a lot.

#!/bin/bash
c=1
while [ $c -le 5000 ]
do
 echo "run $c times"
        ./autopilot/scope-data unity-scope-nearby --results --lat 51.5062348 --long -0.0999525 --set-connected
 (( c++ ))
done

Revision history for this message
Kyle Nitzsche (knitzsche) wrote :

Hi Enwei,

I manually used your click version 1.7 for a while and did not see any panics.

Note: maybe we should not version a dev scope at the final released version (1.7) until it is the REAL 1.7 to prevent confusion. I tend to add a digit for dev versions, like 1.6.1. Then release as 1.7.

thanks.

Revision history for this message
Zhang Enwei (zhangew401) wrote :

Thanks Kyle.
I will change the revision.
But this issue is not totally fixed, it is still reproduced at about once per 3000 tests.
I have written to some golang experts for help, no response till now.

To post a comment you must log in.
This report contains Public information  Edit
Everyone can see this information.

Other bug subscribers