Uploaded image for project: 'MongoDB Database Tools'
  1. MongoDB Database Tools
  2. TOOLS-1429

mongostat panic when monitored server is restarted

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Major - P3
    • Resolution: Fixed
    • Affects Version/s: None
    • Fix Version/s: 3.2.10
    • Component/s: mongostat
    • Labels:
      None
    • Environment:
      OS X 10.11.6
      mongo-tools v3.2.1

      Description

      Didn't try to reproduce but I suspect any variation of the following will do the trick:
      1.) start standalone mongod on 27017
      2.) mongostat 4
      3.) gracefully stop the mongod
      4.) restart mongod after a short while

      Terminal output example

      insert query update delete getmore command flushes mapped vsize     res faults qr|qw ar|aw netIn netOut conn                      time
          10    31     10     *0       0    20|0       0  21.1G 45.0G    1.2G      1   0|0   0|0   21k    91k   42 2016-09-02T09:58:09-04:00
           7     3      3     *0       0     2|0       1  21.1G 45.0G    1.2G      2   0|0   0|0    4k    12k   42 2016-09-02T09:58:13-04:00
           2     5      2     *0       0     9|0       0  21.1G 44.9G    1.2G      0   0|0   0|0    4k    20k   20 2016-09-02T09:58:17-04:00
          *0    *0     *0     *0       0     0|0       0  21.1G 44.9G    1.2G      0   0|0   0|0   56b     8k   17 2016-09-02T09:58:21-04:00
          *0    *0     *0     *0       0     0|0       0  21.1G 44.9G    1.2G      0   0|0   0|0   19b     4k   17 2016-09-02T09:58:25-04:00
          *0    *0     *0     *0       0     2|0       0  21.1G 44.9G    1.2G      0   0|0   0|0  167b     9k   17 2016-09-02T09:58:29-04:00
          *0    *0     *0     *0       0     0|0       0  21.1G 44.9G    1.2G      0   0|0   0|0   19b     4k   17 2016-09-02T09:58:33-04:00
      no reachable servers
      no reachable servers
      no reachable servers
      insert query update delete getmore command % dirty % used flushes   vsize    res qr|qw ar|aw netIn netOut conn                      time
      no reachable servers
      no reachable servers
      no reachable servers
      no reachable servers
      no reachable servers
      no reachable servers
      no reachable servers
      no reachable servers
      panic: runtime error: invalid memory address or nil pointer dereference
      [signal 0xb code=0x1 addr=0x0 pc=0x4055b2b]
       
      goroutine 11 [running]:
      github.com/mongodb/mongo-tools/mongostat.NewStatLine(0xecf5b7c89, 0x90bc0c6, 0x46b2960, 0xc2081941e0, 0x1e, 0xc208130a20, 0x6, 0xc208130a40, 0x6, 0x9d0, ...)
      	/data/mci/src/src/mongo-tools-repo/.gopath/src/github.com/mongodb/mongo-tools/mongostat/stat_types.go:843 +0xc8b
      github.com/mongodb/mongo-tools/mongostat.(*NodeMonitor).Poll(0xc208047090, 0x0, 0x0, 0x4, 0x0)
      	/data/mci/src/src/mongo-tools-repo/.gopath/src/github.com/mongodb/mongo-tools/mongostat/mongostat.go:283 +0xb51
      github.com/mongodb/mongo-tools/mongostat.func·005()
      	/data/mci/src/src/mongo-tools-repo/.gopath/src/github.com/mongodb/mongo-tools/mongostat/mongostat.go:319 +0x201
      created by github.com/mongodb/mongo-tools/mongostat.(*NodeMonitor).Watch
      	/data/mci/src/src/mongo-tools-repo/.gopath/src/github.com/mongodb/mongo-tools/mongostat/mongostat.go:327 +0x16a
       
      goroutine 1 [chan receive, 1418 minutes]:
      github.com/mongodb/mongo-tools/mongostat.(*MongoStat).Run(0xc2080b8060, 0x0, 0x0)
      	/data/mci/src/src/mongo-tools-repo/.gopath/src/github.com/mongodb/mongo-tools/mongostat/mongostat.go:378 +0x185
      main.main()
      	/data/mci/src/src/mongo-tools-repo/mongostat/main/mongostat.go:125 +0xd90
       
      goroutine 5 [syscall, 1418 minutes]:
      os/signal.loop()
      	/usr/local/go/src/os/signal/signal_unix.go:21 +0x1f
      created by os/signal.init·1
      	/usr/local/go/src/os/signal/signal_unix.go:27 +0x35
       
      goroutine 10 [chan receive, 1418 minutes]:
      github.com/mongodb/mongo-tools/common/signals.Handle()
      	/data/mci/src/src/mongo-tools-repo/.gopath/src/github.com/mongodb/mongo-tools/common/signals/signals_unix.go:16 +0x209
      created by main.main
      	/data/mci/src/src/mongo-tools-repo/mongostat/main/mongostat.go:18 +0x34
       
      goroutine 17 [syscall, 1418 minutes, locked to thread]:
      runtime.goexit()
      	/usr/local/go/src/runtime/asm_amd64.s:2232 +0x1
       
      goroutine 13 [sleep]:
      gopkg.in/mgo%2ev2.(*mongoCluster).syncServersLoop(0xc20808a600)
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/cluster.go:368 +0x291
      created by gopkg.in/mgo%2ev2.newCluster
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/cluster.go:78 +0x20b
       
      goroutine 18 [chan receive]:
      github.com/mongodb/mongo-tools/mongostat.func·001()
      	/data/mci/src/src/mongo-tools-repo/.gopath/src/github.com/mongodb/mongo-tools/mongostat/mongostat.go:133 +0x99
      created by github.com/mongodb/mongo-tools/mongostat.(*SyncClusterMonitor).Monitor
      	/data/mci/src/src/mongo-tools-repo/.gopath/src/github.com/mongodb/mongo-tools/mongostat/mongostat.go:148 +0x11c
       
      goroutine 6557 [IO wait]:
      net.(*pollDesc).Wait(0xc2080fb640, 0x72, 0x0, 0x0)
      	/usr/local/go/src/net/fd_poll_runtime.go:84 +0x47
      net.(*pollDesc).WaitRead(0xc2080fb640, 0x0, 0x0)
      	/usr/local/go/src/net/fd_poll_runtime.go:89 +0x43
      net.(*netFD).Read(0xc2080fb5e0, 0xc2081197a0, 0x24, 0x24, 0x0, 0x478bc38, 0xc208178400)
      	/usr/local/go/src/net/fd_unix.go:242 +0x40f
      net.(*conn).Read(0xc20803e4b8, 0xc2081197a0, 0x24, 0x24, 0x0, 0x0, 0x0)
      	/usr/local/go/src/net/net.go:121 +0xdc
      gopkg.in/mgo%2ev2.fill(0x478e398, 0xc20803e4b8, 0xc2081197a0, 0x24, 0x24, 0x0, 0x0)
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/socket.go:530 +0x75
      gopkg.in/mgo%2ev2.(*mongoSocket).readLoop(0xc2081d5420)
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/socket.go:547 +0x126
      created by gopkg.in/mgo%2ev2.newSocket
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/socket.go:194 +0x3d3
       
      goroutine 6538 [sleep]:
      gopkg.in/mgo%2ev2.(*mongoServer).pinger(0xc208120d20, 0x1)
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/server.go:296 +0x169
      created by gopkg.in/mgo%2ev2.newServer
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/server.go:89 +0x194
       
      goroutine 1030 [IO wait]:
      net.(*pollDesc).Wait(0xc20823ad80, 0x77, 0x0, 0x0)
      	/usr/local/go/src/net/fd_poll_runtime.go:84 +0x47
      net.(*pollDesc).WaitWrite(0xc20823ad80, 0x0, 0x0)
      	/usr/local/go/src/net/fd_poll_runtime.go:93 +0x43
      net.(*netFD).connect(0xc20823ad20, 0x0, 0x0, 0x478d358, 0xc20824eca0, 0xecf5b7de1, 0x39df7b4b, 0x46b2960, 0x0, 0x0)
      	/usr/local/go/src/net/fd_unix.go:114 +0x24b
      net.(*netFD).dial(0xc20823ad20, 0x478e3f8, 0x0, 0x478e3f8, 0xc2081195c0, 0xecf5b7de1, 0x39df7b4b, 0x46b2960, 0x0, 0x0)
      	/usr/local/go/src/net/sock_posix.go:139 +0x37a
      net.socket(0x4480910, 0x3, 0x2, 0x1, 0x0, 0xc208119500, 0x478e3f8, 0x0, 0x478e3f8, 0xc2081195c0, ...)
      	/usr/local/go/src/net/sock_posix.go:91 +0x422
      net.internetSocket(0x4480910, 0x3, 0x478e3f8, 0x0, 0x478e3f8, 0xc2081195c0, 0xecf5b7de1, 0x39df7b4b, 0x46b2960, 0x1, ...)
      	/usr/local/go/src/net/ipsock_posix.go:137 +0x148
      net.dialTCP(0x4480910, 0x3, 0x0, 0xc2081195c0, 0xecf5b7de1, 0x39df7b4b, 0x46b2960, 0xc20822ff90, 0x0, 0x0)
      	/usr/local/go/src/net/tcpsock_posix.go:156 +0x125
      net.dialSingle(0x4480910, 0x3, 0xc20823c480, 0xf, 0x0, 0x0, 0x478e368, 0xc2081195c0, 0xecf5b7de1, 0x39df7b4b, ...)
      	/usr/local/go/src/net/dial.go:235 +0x3f5
      net.func·016(0xecf5b7de1, 0xe39df7b4b, 0x46b2960, 0x0, 0x0, 0x0, 0x0)
      	/usr/local/go/src/net/dial.go:163 +0x12c
      net.dial(0x4480910, 0x3, 0x478e368, 0xc2081195c0, 0xc20817fa98, 0xecf5b7de1, 0xe39df7b4b, 0x46b2960, 0x0, 0x0, ...)
      	/usr/local/go/src/net/fd_unix.go:40 +0x6f
      net.(*Dialer).Dial(0xc20824dc40, 0x4480910, 0x3, 0xc20823c480, 0xf, 0x0, 0x0, 0x0, 0x0)
      	/usr/local/go/src/net/dial.go:170 +0x4be
      net.DialTimeout(0x4480910, 0x3, 0xc20823c480, 0xf, 0x37e11d600, 0x0, 0x0, 0x0, 0x0)
      	/usr/local/go/src/net/dial.go:150 +0xe6
      gopkg.in/mgo%2ev2.(*mongoServer).Connect(0xc2082320e0, 0x37e11d600, 0xc208164900, 0x0, 0x0)
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/server.go:164 +0x295
      gopkg.in/mgo%2ev2.(*mongoServer).AcquireSocket(0xc2082320e0, 0x0, 0x37e11d600, 0xc20824ec00, 0x37864d00, 0x0, 0x0)
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/server.go:129 +0x278
      gopkg.in/mgo%2ev2.(*mongoServer).pinger(0xc2082320e0, 0x1)
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/server.go:299 +0x1e4
      created by gopkg.in/mgo%2ev2.newServer
      	/data/mci/src/src/mongo-tools-repo/vendor/src/gopkg.in/mgo.v2/server.go:89 +0x194
      

        Attachments

          Activity

            People

            Assignee:
            gabriel.russell Gabriel Russell (Inactive)
            Reporter:
            john.morales John Morales
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

              Dates

              Created:
              Updated:
              Resolved: