[SERVER-4367] segfault in mongos Created: 24/Nov/11  Updated: 06/Apr/23  Resolved: 15/Feb/12

Status: Closed
Project: Core Server
Component/s: Sharding
Affects Version/s: 2.0.1
Fix Version/s: 2.1.1

Type: Bug Priority: Major - P3
Reporter: Zac Witte Assignee: Mathias Stearn
Resolution: Done Votes: 0
Labels: None
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified
Environment:

Ubuntu 11.04 on Amazon EC2


Operating System: Linux
Participants:

 Description   

Quit mongos with Ctrl+C and received the following stack trace.

Wed Nov 23 23:58:34 [mongosMain] dbexit: received signal 2 rc:0 received signal 2

      • glibc detected *** mongos: free(): invalid pointer: 0x0000000000b0d380 ***
        ======= Backtrace: =========
        /lib/x86_64-linux-gnu/libc.so.6(+0x78a8f)[0x7fd2c1cada8f]
        /lib/x86_64-linux-gnu/libc.so.6(cfree+0x73)[0x7fd2c1cb18e3]
        mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0x1b9)[0x529a29]
        mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runEv+0x74)[0x52b9e4]
        mongos(thread_proxy+0x80)[0x7feeb0]
        /lib/x86_64-linux-gnu/libpthread.so.0(+0x6d8c)[0x7fd2c2770d8c]
        /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7fd2c1d1b04d]
        ======= Memory map: ========
        00400000-008f4000 r-xp 00000000 ca:01 302 /usr/bin/mongos
        00af3000-00b0a000 rw-p 004f3000 ca:01 302 /usr/bin/mongos
        00b0a000-00b0e000 rw-p 00000000 00:00 0
        0141e000-01759000 rw-p 00000000 00:00 0 [heap]
        7fd2b77f7000-7fd2b77f8000 ---p 00000000 00:00 0
        7fd2b77f8000-7fd2b7ff8000 rw-p 00000000 00:00 0
        7fd2b7ff8000-7fd2b7ff9000 ---p 00000000 00:00 0
        7fd2b7ff9000-7fd2b87f9000 rw-p 00000000 00:00 0
        7fd2b87f9000-7fd2b87fa000 ---p 00000000 00:00 0
        7fd2b87fa000-7fd2b8ffa000 rw-p 00000000 00:00 0
        7fd2b8ffa000-7fd2b8ffb000 ---p 00000000 00:00 0
        7fd2b8ffb000-7fd2b97fb000 rw-p 00000000 00:00 0
        7fd2b97fb000-7fd2b97fc000 ---p 00000000 00:00 0
        7fd2b97fc000-7fd2b9ffc000 rw-p 00000000 00:00 0
        7fd2b9ffc000-7fd2b9ffd000 ---p 00000000 00:00 0
        7fd2b9ffd000-7fd2ba7fd000 rw-p 00000000 00:00 0
        7fd2ba7fd000-7fd2ba7fe000 ---p 00000000 00:00 0
        7fd2ba7fe000-7fd2baffe000 rw-p 00000000 00:00 0
        7fd2baffe000-7fd2bafff000 ---p 00000000 00:00 0
        7fd2bafff000-7fd2bb7ff000 rw-p 00000000 00:00 0
        7fd2bb7ff000-7fd2bb800000 ---p 00000000 00:00 0
        7fd2bb800000-7fd2bc000000 rw-p 00000000 00:00 0
        7fd2bc000000-7fd2bc115000 rw-p 00000000 00:00 0
        7fd2bc115000-7fd2c0000000 ---p 00000000 00:00 0
        7fd2c01ff000-7fd2c0200000 ---p 00000000 00:00 0
        7fd2c0200000-7fd2c0300000 rw-p 00000000 00:00 0
        7fd2c0300000-7fd2c0301000 ---p 00000000 00:00 0
        7fd2c0301000-7fd2c0401000 rw-p 00000000 00:00 0
        7fd2c0401000-7fd2c0402000 ---p 00000000 00:00 0
        7fd2c0402000-7fd2c0502000 rw-p 00000000 00:00 0
        7fd2c0502000-7fd2c0503000 ---p 00000000 00:00 0
        7fd2c0503000-7fd2c0603000 rw-p 00000000 00:00 0
        7fd2c0603000-7fd2c0604000 ---p 00000000 00:00 0
        7fd2c0604000-7fd2c0e04000 rw-p 00000000 00:00 0
        7fd2c0e04000-7fd2c0e05000 ---p 00000000 00:00 0
        7fd2c0e05000-7fd2c1605000 rw-p 00000000 00:00 0
        7fd2c1605000-7fd2c161c000 r-xp 00000000 ca:01 395690 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7fd2c161c000-7fd2c181c000 ---p 00017000 ca:01 395690 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7fd2c181c000-7fd2c181d000 r--p 00017000 ca:01 395690 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7fd2c181d000-7fd2c181e000 rw-p 00018000 ca:01 395690 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7fd2c181e000-7fd2c1820000 rw-p 00000000 00:00 0
        7fd2c1820000-7fd2c1827000 r-xp 00000000 ca:01 395678 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7fd2c1827000-7fd2c1a26000 ---p 00007000 ca:01 395678 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7fd2c1a26000-7fd2c1a27000 r--p 00006000 ca:01 395678 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7fd2c1a27000-7fd2c1a28000 rw-p 00007000 ca:01 395678 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7fd2c1a28000-7fd2c1a34000 r-xp 00000000 ca:01 395679 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7fd2c1a34000-7fd2c1c33000 ---p 0000c000 ca:01 395679 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7fd2c1c33000-7fd2c1c34000 r--p 0000b000 ca:01 395679 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7fd2c1c34000-7fd2c1c35000 rw-p 0000c000 ca:01 395679 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7fd2c1c35000-7fd2c1dbf000 r-xp 00000000 ca:01 395655 /lib/x86_64-linux-gnu/libc-2.13.so
        7fd2c1dbf000-7fd2c1fbe000 ---p 0018a000 ca:01 395655 /lib/x86_64-linux-gnu/libc-2.13.so
        7fd2c1fbe000-7fd2c1fc2000 r--p 00189000 ca:01 395655 /lib/x86_64-linux-gnu/libc-2.13.so
        7fd2c1fc2000-7fd2c1fc3000 rw-p 0018d000 ca:01 395655 /lib/x86_64-linux-gnu/libc-2.13.so
        7fd2c1fc3000-7fd2c1fc9000 rw-p 00000000 00:00 0
        7fd2c1fc9000-7fd2c1fde000 r-xp 00000000 ca:01 395669 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7fd2c1fde000-7fd2c21dd000 ---p 00015000 ca:01 395669 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7fd2c21dd000-7fd2c21de000 r--p 00014000 ca:01 395669 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7fd2c21de000-7fd2c21df000 rw-p 00015000 ca:01 395669 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7fd2c21df000-7fd2c2263000 r-xp 00000000 ca:01 395674 /lib/x86_64-linux-gnu/libm-2.13.so
        7fd2c2263000-7fd2c2462000 ---p 00084000 ca:01 395674 /lib/x86_64-linux-gnu/libm-2.13.so
        7fd2c2462000-7fd2c2463000 r--p 00083000 ca:01 395674 /lib/x86_64-linux-gnu/libm-2.13.so
        7fd2c2463000-7fd2c2464000 rw-p 00084000 ca:01 395674 /lib/x86_64-linux-gnu/libm-2.13.so
        7fd2c2464000-7fd2c254c000 r-xp 00000000 ca:01 8380 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.14
        7fd2c254c000-7fd2c274b000 ---p 000e8000 ca:01 8380 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.14
        7fd2c274b000-7fd2c2753000 r--p 000e7000 ca:01 8380 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.14
        7fd2c2753000-7fd2c2755000 rw-p 000ef000 ca:01 8380 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.14
        7fd2c2755000-7fd2c276a000 rw-p 00000000 00:00 0
        7fd2c276a000-7fd2c2782000 r-xp 00000000 ca:01 395689 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7fd2c2782000-7fd2c2982000 ---p 00018000 ca:01 395689 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7fd2c2982000-7fd2c2983000 r--p 00018000 ca:01 395689 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7fd2c2983000-7fd2c2984000 rw-p 00019000 ca:01 395689 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7fd2c2984000-7fd2c2988000 rw-p 00000000 00:00 0
        7fd2c2988000-7fd2c29a9000 r-xp 00000000 ca:01 395648 /lib/x86_64-linux-gnu/ld-2.13.so
        7fd2c2a9b000-7fd2c2a9c000 ---p 00000000 00:00 0
        7fd2c2a9c000-7fd2c2ba1000 rw-p 00000000 00:00 0
        7fd2c2ba5000-7fd2c2ba8000 rw-p 00000000 00:00 0
        7fd2c2ba8000-7fd2c2ba9000 r--p 00020000 ca:01 395648 /lib/x86_64-linux-gnu/ld-2.13.so
        7fd2c2ba9000-7fd2c2bab000 rw-p 00021000 ca:01 395648 /lib/x86_64-linux-gnu/ld-2.13.so
        7fff43e91000-7fff43eb2000 rw-p 00000000 00:00 0 [stack]
        7fff43ff2000-7fff43ff3000 r-xp 00000000 00:00 0 [vdso]Received signal 6
        Backtrace: 0x5521f5 0x7fd2c1c68d80 0x7fd2c1c68d05 0x7fd2c1c6cab6 0x7fd2c1ca1d7b 0x7fd2c1cada8f 0x7fd2c1cb18e3 0x529a29 0x52b9e4 0x7feeb0 0x7fd2c2770d8c 0x7fd2c1d1b04d
        mongos(_ZN5mongo17printStackAndExitEi+0x75)[0x5521f5]
        /lib/x86_64-linux-gnu/libc.so.6(+0x33d80)[0x7fd2c1c68d80]
        /lib/x86_64-linux-gnu/libc.so.6(gsignal+0x35)[0x7fd2c1c68d05]
        /lib/x86_64-linux-gnu/libc.so.6(abort+0x186)[0x7fd2c1c6cab6]
        /lib/x86_64-linux-gnu/libc.so.6(+0x6cd7b)[0x7fd2c1ca1d7b]
        /lib/x86_64-linux-gnu/libc.so.6(+0x78a8f)[0x7fd2c1cada8f]
        /lib/x86_64-linux-gnu/libc.so.6(cfree+0x73)[0x7fd2c1cb18e3]
        mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0x1b9)[0x529a29]
        mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runEv+0x74)[0x52b9e4]
        mongos(thread_proxy+0x80)[0x7feeb0]
        /lib/x86_64-linux-gnu/libpthread.so.0(+0x6d8c)[0x7fd2c2770d8c]
        /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7fd2c1d1b04d]
        ===
        Logstream::get called in uninitialized state
        Wed Nov 23 23:58:45 [LockPinger] warning: removing distributed lock ping thread 'mongo2.pubnub.com:27019/mongo1:27017:1322031244:1804289383'
        Logstream::get called in uninitialized state


 Comments   
Comment by Zac Witte [ 12/Jun/12 ]

I would also really like this to be backported. I get the segfault nearly every time I try to stop mongos.

Comment by Adam Flynn [ 04/Jun/12 ]

Any chance of a backport on this one? I get it frequently when I try to restart mongos (I'm running ~75 of them). Makes automated config changes incredibly difficult since I have to manually kill -9 a bunch of them (and causes app-level problems when I end up with mongos in a busted state or have to sigkill it).

Comment by Mathias Stearn [ 15/Feb/12 ]

There are two separate causes of segfaults in this ticket, both have been patched in master. Please open a new ticket if you find another cause.

Comment by Mathias Stearn [ 14/Feb/12 ]

That last commit (max macro) was actually for SERVER-4834 not this ticket

Comment by auto [ 14/Feb/12 ]

Author:

{u'login': u'RedBeard0531', u'name': u'Mathias Stearn', u'email': u'mathias@10gen.com'}

Message: prevent windows from treating max as a macro SERVER-4367
Branch: master
https://github.com/mongodb/mongo/commit/a0e38678ffd22d9a2f428e27b603b6380838276d

Comment by auto [ 13/Feb/12 ]

Author:

{u'login': u'RedBeard0531', u'name': u'Mathias Stearn', u'email': u'mathias@10gen.com'}

Message: Use Task API correctly in mongos SERVER-4367
Branch: master
https://github.com/mongodb/mongo/commit/5f3e16f58d35e2ea21bff282d17224061527af98

Comment by auto [ 09/Feb/12 ]

Author:

{u'login': u'RedBeard0531', u'name': u'Mathias Stearn', u'email': u'mathias@10gen.com'}

Message: Don't destroy _hostNameCached SERVER-4367
Branch: master
https://github.com/mongodb/mongo/commit/af663932ba6aeb3e17ed298e971b68ade9286e19

Comment by Zac Witte [ 08/Feb/12 ]

Only when hitting ctrl+c. When it happens it never actually shuts down, though. I have to kill it with kill -9.

Comment by Mathias Stearn [ 08/Feb/12 ]

Are you seeing this while running or only when you try to shutdown with ctrl-c?

Comment by Zac Witte [ 07/Feb/12 ]

Got it again. This time there were no problems with any shards, but it's possible that balancing as in progress.

Tue Feb 7 06:11:40 [Balancer] ChunkManager: time to load chunks for pb3.hourly_stats: 28ms sequenceNumber: 95 version: 212|1
Tue Feb 7 06:11:40 [Balancer] distributed lock 'balancer/mongo1:27017:1328584620:1804289383' unlocked.
^CTue Feb 7 06:13:02 [mongosMain] dbexit: received signal 2 rc:0 received signal 2

      • glibc detected *** bin/mongos: free(): invalid pointer: 0x0000000000b15360 ***
        ======= Backtrace: =========
        /lib/x86_64-linux-gnu/libc.so.6(+0x78a96)[0x7f257d797a96]
        /lib/x86_64-linux-gnu/libc.so.6(cfree+0x6c)[0x7f257d79bd7c]
        bin/mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0x1b9)[0x5256c9]
        bin/mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runEv+0x74)[0x527684]
        bin/mongos(thread_proxy+0x80)[0x804350]
        /lib/x86_64-linux-gnu/libpthread.so.0(+0x7efc)[0x7f257e266efc]
        /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f257d80089d]
        ======= Memory map: ========
        00400000-008fb000 r-xp 00000000 ca:01 263147 /home/ubuntu/mongodb-linux-x86_64-2.0.2/bin/mongos
        00afb000-00b12000 rw-p 004fb000 ca:01 263147 /home/ubuntu/mongodb-linux-x86_64-2.0.2/bin/mongos
        00b12000-00b16000 rw-p 00000000 00:00 0
        010fe000-01178000 rw-p 00000000 00:00 0 [heap]
        7f2573ef7000-7f2573ef8000 ---p 00000000 00:00 0
        7f2573ef8000-7f2573ff8000 rw-p 00000000 00:00 0
        7f2573ff8000-7f2573ff9000 ---p 00000000 00:00 0
        7f2573ff9000-7f25747f9000 rw-p 00000000 00:00 0
        7f25747f9000-7f25747fa000 ---p 00000000 00:00 0
        7f25747fa000-7f2574ffa000 rw-p 00000000 00:00 0
        7f2574ffa000-7f2574ffb000 ---p 00000000 00:00 0
        7f2574ffb000-7f25757fb000 rw-p 00000000 00:00 0
        7f25757fb000-7f25757fc000 ---p 00000000 00:00 0
        7f25757fc000-7f2575ffc000 rw-p 00000000 00:00 0
        7f2575ffc000-7f2575ffd000 ---p 00000000 00:00 0
        7f2575ffd000-7f25767fd000 rw-p 00000000 00:00 0
        7f25767fd000-7f25767fe000 ---p 00000000 00:00 0
        7f25767fe000-7f2576ffe000 rw-p 00000000 00:00 0
        7f2576ffe000-7f2576fff000 ---p 00000000 00:00 0
        7f2576fff000-7f25777ff000 rw-p 00000000 00:00 0
        7f25777ff000-7f2577800000 ---p 00000000 00:00 0
        7f2577800000-7f2578000000 rw-p 00000000 00:00 0
        7f2578000000-7f257834b000 rw-p 00000000 00:00 0
        7f257834b000-7f257c000000 ---p 00000000 00:00 0
        7f257c0ee000-7f257c0ef000 ---p 00000000 00:00 0
        7f257c0ef000-7f257c8ef000 rw-p 00000000 00:00 0
        7f257c8ef000-7f257c8f0000 ---p 00000000 00:00 0
        7f257c8f0000-7f257d0f0000 rw-p 00000000 00:00 0
        7f257d0f0000-7f257d107000 r-xp 00000000 ca:01 395728 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7f257d107000-7f257d307000 ---p 00017000 ca:01 395728 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7f257d307000-7f257d308000 r--p 00017000 ca:01 395728 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7f257d308000-7f257d309000 rw-p 00018000 ca:01 395728 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7f257d309000-7f257d30b000 rw-p 00000000 00:00 0
        7f257d30b000-7f257d311000 r-xp 00000000 ca:01 395729 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7f257d311000-7f257d510000 ---p 00006000 ca:01 395729 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7f257d510000-7f257d511000 r--p 00005000 ca:01 395729 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7f257d511000-7f257d512000 rw-p 00006000 ca:01 395729 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7f257d512000-7f257d51e000 r-xp 00000000 ca:01 395727 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7f257d51e000-7f257d71d000 ---p 0000c000 ca:01 395727 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7f257d71d000-7f257d71e000 r--p 0000b000 ca:01 395727 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7f257d71e000-7f257d71f000 rw-p 0000c000 ca:01 395727 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7f257d71f000-7f257d8b4000 r-xp 00000000 ca:01 395724 /lib/x86_64-linux-gnu/libc-2.13.so
        7f257d8b4000-7f257dab3000 ---p 00195000 ca:01 395724 /lib/x86_64-linux-gnu/libc-2.13.so
        7f257dab3000-7f257dab7000 r--p 00194000 ca:01 395724 /lib/x86_64-linux-gnu/libc-2.13.so
        7f257dab7000-7f257dab8000 rw-p 00198000 ca:01 395724 /lib/x86_64-linux-gnu/libc-2.13.so
        7f257dab8000-7f257dabe000 rw-p 00000000 00:00 0
        7f257dabe000-7f257dad3000 r-xp 00000000 ca:01 395714 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7f257dad3000-7f257dcd2000 ---p 00015000 ca:01 395714 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7f257dcd2000-7f257dcd3000 r--p 00014000 ca:01 395714 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7f257dcd3000-7f257dcd4000 rw-p 00015000 ca:01 395714 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7f257dcd4000-7f257dd57000 r-xp 00000000 ca:01 395734 /lib/x86_64-linux-gnu/libm-2.13.so
        7f257dd57000-7f257df56000 ---p 00083000 ca:01 395734 /lib/x86_64-linux-gnu/libm-2.13.so
        7f257df56000-7f257df57000 r--p 00082000 ca:01 395734 /lib/x86_64-linux-gnu/libm-2.13.so
        7f257df57000-7f257df58000 rw-p 00083000 ca:01 395734 /lib/x86_64-linux-gnu/libm-2.13.so
        7f257df58000-7f257e040000 r-xp 00000000 ca:01 10571 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
        7f257e040000-7f257e240000 ---p 000e8000 ca:01 10571 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
        7f257e240000-7f257e248000 r--p 000e8000 ca:01 10571 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
        7f257e248000-7f257e24a000 rw-p 000f0000 ca:01 10571 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
        7f257e24a000-7f257e25f000 rw-p 00000000 00:00 0
        7f257e25f000-7f257e277000 r-xp 00000000 ca:01 395732 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7f257e277000-7f257e476000 ---p 00018000 ca:01 395732 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7f257e476000-7f257e477000 r--p 00017000 ca:01 395732 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7f257e477000-7f257e478000 rw-p 00018000 ca:01 395732 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7f257e478000-7f257e47c000 rw-p 00000000 00:00 0
        7f257e47c000-7f257e49d000 r-xp 00000000 ca:01 395731 /lib/x86_64-linux-gnu/ld-2.13.so
        7f257e58e000-7f257e58f000 ---p 00000000 00:00 0
        7f257e58f000-7f257e694000 rw-p 00000000 00:00 0
        7f257e699000-7f257e69c000 rw-p 00000000 00:00 0
        7f257e69c000-7f257e69d000 r--p 00020000 ca:01 395731 /lib/x86_64-linux-gnu/ld-2.13.so
        7f257e69d000-7f257e69f000 rw-p 00021000 ca:01 395731 /lib/x86_64-linux-gnu/ld-2.13.so
        7fff944fb000-7fff9451c000 rw-p 00000000 00:00 0 [stack]
        7fff945a6000-7fff945a7000 r-xp 00000000 00:00 0 [vdso]
        ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
        Received signal 6
        Backtrace: 0x54de95 0x7f257d755420 0x7f257d7553a5 0x7f257d758b0b 0x7f257d78d113 0x7f257d797a96 0x7f257d79bd7c 0x5256c9 0x527684 0x804350 0x7f257e266efc 0x7f257d80089d
        bin/mongos(_ZN5mongo17printStackAndExitEi+0x75)[0x54de95]
        /lib/x86_64-linux-gnu/libc.so.6(+0x36420)[0x7f257d755420]
        /lib/x86_64-linux-gnu/libc.so.6(gsignal+0x35)[0x7f257d7553a5]
        /lib/x86_64-linux-gnu/libc.so.6(abort+0x17b)[0x7f257d758b0b]
        /lib/x86_64-linux-gnu/libc.so.6(+0x6e113)[0x7f257d78d113]
        /lib/x86_64-linux-gnu/libc.so.6(+0x78a96)[0x7f257d797a96]
        /lib/x86_64-linux-gnu/libc.so.6(cfree+0x6c)[0x7f257d79bd7c]
        bin/mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0x1b9)[0x5256c9]
        bin/mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runEv+0x74)[0x527684]
        bin/mongos(thread_proxy+0x80)[0x804350]
        /lib/x86_64-linux-gnu/libpthread.so.0(+0x7efc)[0x7f257e266efc]
        /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f257d80089d]
        ===
        Logstream::get called in uninitialized state
        Tue Feb 7 06:13:12 [LockPinger] warning: removing distributed lock ping thread 'mongo1.foobar.com:27019,mongo2.foobar.com:27019,mongo3.foobar.com:27019/mongo1:27017:1328584620:1804289383'
Comment by Mathias Stearn [ 06/Feb/12 ]

Ok, I think I see one possible issue here. Will have a fix soon

Comment by Zac Witte [ 05/Feb/12 ]

Ran into it again. This happened when one of the shards stopped responding. I then tried to shut down the cluster and this exception occurred when I sent the kill signal with Ctrl+c on mongos.

Sun Feb 5 04:25:26 [conn1915] CMD fsync: sync:1 lock:0
Sun Feb 5 04:25:29 [conn1914] end connection xxx.xxx.xxx.xxx:46702
Sun Feb 5 04:25:29 [conn1915] end connection xxx.xxx.xxx.xxx:53524
Sun Feb 5 04:25:32 [initandlisten] connection accepted from xxx.xxx.xxx.xxx:39845 #1916
Sun Feb 5 04:25:32 [conn1913] end connection xxx.xxx.xxx.xxx:46697
Sun Feb 5 04:25:32 [conn1916] CMD fsync: sync:1 lock:0
Sun Feb 5 04:25:35 [initandlisten] connection accepted from xxx.xxx.xxx.xxx:46708 #1917
Sun Feb 5 04:25:44 [conn1910] end connection xxx.xxx.xxx.xxx:46693
Sun Feb 5 04:25:44 [conn1907] end connection xxx.xxx.xxx.xxx:41363
Sun Feb 5 04:25:35 [WriteBackListener-mongo3.foobar.com:27018] SyncClusterConnection connecting to [mongo2.foobar.com:27019]
Sun Feb 5 04:25:35 [WriteBackListener-mongo3.foobar.com:27018] SyncClusterConnection connecting to [mongo3.foobar.com:27019]
Sun Feb 5 04:25:41 [CheckConfigServers] warning: couldn't check on config server:mongo3.foobar.com:27019 ok for now : 11002 socket exception [6] server [mongo3.foobar.com:27019] mongos connectionpool error: couldn't connect to server mongo3.foobar.com:27019
Sun Feb 5 04:25:44 [WriteBackListener-mongo3.foobar.com:27018] SyncClusterConnection connect fail to: mongo3.foobar.com:27019 errmsg: couldn't connect to server mongo3.foobar.com:27019
Sun Feb 5 04:25:44 [Balancer] ~ScopedDbConnection: _conn != null
Sun Feb 5 04:25:44 [Balancer] caught exception while doing balance: socket exception
Sun Feb 5 04:25:47 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:25:50 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:25:53 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:25:59 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:26:02 [LockPinger] SyncClusterConnection connecting to [mongo1.foobar.com:27019]
Sun Feb 5 04:26:02 [LockPinger] SyncClusterConnection connecting to [mongo2.foobar.com:27019]
Sun Feb 5 04:26:02 [LockPinger] SyncClusterConnection connecting to [mongo3.foobar.com:27019]
Sun Feb 5 04:26:02 [LockPinger] SyncClusterConnection connect fail to: mongo3.foobar.com:27019 errmsg: couldn't connect to server mongo3.foobar.com:27019
Sun Feb 5 04:26:02 [LockPinger] trying reconnect to mongo3.foobar.com:27019
Sun Feb 5 04:26:05 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:26:08 [LockPinger] reconnect mongo3.foobar.com:27019 failed couldn't connect to server mongo3.foobar.com:27019
Sun Feb 5 04:26:08 [LockPinger] ~ScopedDbConnection: _conn != null
Sun Feb 5 04:26:08 [LockPinger] warning: distributed lock pinger 'mongo1.foobar.com:27019,mongo2.foobar.com:27019,mongo3.foobar.com:27019/mongo1:27017:1328338177:1804289383' detected an exception while pinging. :: caused by :: SyncClusterConnection::udpate prepare failed: 9001 socket exception [6] server [mongo3.foobar.com:27019] mongo3.foobar.com:27019:{}
Sun Feb 5 04:26:11 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:26:14 [Balancer] trying reconnect to mongo3.foobar.com:27019
Sun Feb 5 04:26:14 [Balancer] reconnect mongo3.foobar.com:27019 failed couldn't connect to server mongo3.foobar.com:27019
Sun Feb 5 04:26:17 [Balancer] ~ScopedDbConnection: _conn != null
Sun Feb 5 04:26:17 [Balancer] caught exception while doing balance: socket exception
Sun Feb 5 04:26:20 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:26:20 [WriteBackListener-mongo3.foobar.com:27018] SyncClusterConnection connecting to [mongo1.foobar.com:27019]
Sun Feb 5 04:26:20 [WriteBackListener-mongo3.foobar.com:27018] SyncClusterConnection connecting to [mongo2.foobar.com:27019]
Sun Feb 5 04:26:20 [WriteBackListener-mongo3.foobar.com:27018] SyncClusterConnection connecting to [mongo3.foobar.com:27019]
Sun Feb 5 04:26:26 [WriteBackListener-mongo3.foobar.com:27018] SyncClusterConnection connect fail to: mongo3.foobar.com:27019 errmsg: couldn't connect to server mongo3.foobar.com:27019
Sun Feb 5 04:26:35 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:26:38 [LockPinger] SyncClusterConnection connecting to [mongo1.foobar.com:27019]
Sun Feb 5 04:26:38 [LockPinger] SyncClusterConnection connecting to [mongo2.foobar.com:27019]
Sun Feb 5 04:26:38 [LockPinger] SyncClusterConnection connecting to [mongo3.foobar.com:27019]
Sun Feb 5 04:26:41 [CheckConfigServers] warning: couldn't check on config server:mongo3.foobar.com:27019 ok for now : 11002 socket exception [6] server [mongo3.foobar.com:27019] mongos connectionpool error: couldn't connect to server mongo3.foobar.com:27019
Sun Feb 5 04:26:44 [WriteBackListener-mongo3.foobar.com:27018] WriteBackListener exception : socket exception
Sun Feb 5 04:26:44 [WriteBackListener-mongo3.foobar.com:27018] DBClientCursor::init call() failed
Sun Feb 5 04:26:44 [WriteBackListener-mongo3.foobar.com:27018] query failed to: mongo1.foobar.com:27019 no data
Sun Feb 5 04:26:47 [Balancer] trying reconnect to mongo1.foobar.com:27019
Sun Feb 5 04:26:47 [Balancer] reconnect mongo1.foobar.com:27019 failed couldn't connect to server mongo1.foobar.com:27019
Sun Feb 5 04:26:47 [Balancer] trying reconnect to mongo3.foobar.com:27019
^CSun Feb 5 04:26:49 [mongosMain] dbexit: received signal 2 rc:0 received signal 2

      • glibc detected *** bin/mongos: free(): invalid pointer: 0x0000000000b15360 ***
        Received signal 11
        Backtrace: ======= Backtrace: =========
        /lib/x86_64-linux-gnu/libc.so.6(+0x78a96)[0x7f90e332ea96]
        /lib/x86_64-linux-gnu/libc.so.6(cfree+0x6c)[0x7f90e3332d7c]
        bin/mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0x1b9)[0x5256c9]
        bin/mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runEv+0x74)[0x527684]
        bin/mongos(thread_proxy+0x80)[0x804350]
        /lib/x86_64-linux-gnu/libpthread.so.0(+0x7efc)[0x7f90e3dfdefc]
        /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f90e339789d]
        ======= Memory map: ========
        00400000-008fb000 r-xp 00000000 ca:01 263147 /home/ubuntu/mongodb-linux-x86_64-2.0.2/bin/mongos
        00afb000-00b12000 rw-p 004fb000 ca:01 263147 /home/ubuntu/mongodb-linux-x86_64-2.0.2/bin/mongos
        00b12000-00b16000 rw-p 00000000 00:00 0
        0169d000-0179f000 rw-p 00000000 00:00 0 [heap]
        7f90d6930000-7f90d7ff8000 rw-p 00000000 00:00 0
        7f90d7ff8000-7f90d7ff9000 ---p 00000000 00:00 0
        7f90d7ff9000-7f90d87f9000 rw-p 00000000 00:00 0
        7f90d87f9000-7f90d87fa000 ---p 00000000 00:00 0
        7f90d87fa000-7f90d8ffa000 rw-p 00000000 00:00 0
        7f90d8ffa000-7f90d8ffb000 ---p 00000000 00:00 0
        7f90d8ffb000-7f90d97fb000 rw-p 00000000 00:00 0
        7f90d97fb000-7f90d97fc000 ---p 00000000 00:00 0
        7f90d97fc000-7f90d9ffc000 rw-p 00000000 00:00 0
        7f90d9ffc000-7f90d9ffd000 ---p 00000000 00:00 0
        7f90d9ffd000-7f90da7fd000 rw-p 00000000 00:00 0
        7f90da7fd000-7f90da7fe000 ---p 00000000 00:00 0
        7f90da7fe000-7f90daffe000 rw-p 00000000 00:00 0
        7f90daffe000-7f90dafff000 ---p 00000000 00:00 0
        7f90dafff000-7f90db7ff000 rw-p 00000000 00:00 0
        7f90db7ff000-7f90db800000 ---p 00000000 00:00 0
        7f90db800000-7f90dc000000 rw-p 00000000 00:00 0
        7f90dc000000-7f90dc26b000 rw-p 00000000 00:00 0
        7f90dc26b000-7f90e0000000 ---p 00000000 00:00 0
        7f90e017f000-7f90e0180000 ---p 00000000 00:00 0
        7f90e0180000-7f90e0280000 rw-p 00000000 00:00 0
        7f90e0280000-7f90e0281000 ---p 00000000 00:00 0
        7f90e0281000-7f90e0381000 rw-p 00000000 00:00 0
        7f90e0381000-7f90e0382000 ---p 00000000 00:00 0
        7f90e0382000-7f90e0482000 rw-p 00000000 00:00 0
        7f90e0482000-7f90e0483000 ---p 00000000 00:00 0
        7f90e0483000-7f90e0c83000 rw-p 00000000 00:00 0
        7f90e0c83000-7f90e0c84000 ---p 00000000 00:00 0
        7f90e0c84000-7f90e1484000 rw-p 00000000 00:00 0
        7f90e1484000-7f90e1485000 ---p 00000000 00:00 0
        7f90e1485000-7f90e1c85000 rw-p 00000000 00:00 0
        7f90e1c85000-7f90e1c86000 ---p 00000000 00:00 0
        7f90e1c86000-7f90e2486000 rw-p 00000000 00:00 0
        7f90e2486000-7f90e2487000 ---p 00000000 00:00 0
        7f90e2487000-7f90e2c87000 rw-p 00000000 00:00 0
        7f90e2c87000-7f90e2c9e000 r-xp 00000000 ca:01 395728 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7f90e2c9e000-7f90e2e9e000 ---p 00017000 ca:01 395728 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7f90e2e9e000-7f90e2e9f000 r--p 00017000 ca:01 395728 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7f90e2e9f000-7f90e2ea0000 rw-p 00018000 ca:01 395728 /lib/x86_64-linux-gnu/libresolv-2.13.so
        7f90e2ea0000-7f90e2ea2000 rw-p 00000000 00:00 0
        7f90e2ea2000-7f90e2ea8000 r-xp 00000000 ca:01 395729 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7f90e2ea8000-7f90e30a7000 ---p 00006000 ca:01 395729 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7f90e30a7000-7f90e30a8000 r--p 00005000 ca:01 395729 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7f90e30a8000-7f90e30a9000 rw-p 00006000 ca:01 395729 /lib/x86_64-linux-gnu/libnss_dns-2.13.so
        7f90e30a9000-7f90e30b5000 r-xp 00000000 ca:01 395727 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7f90e30b5000-7f90e32b4000 ---p 0000c000 ca:01 395727 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7f90e32b4000-7f90e32b5000 r--p 0000b000 ca:01 395727 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7f90e32b5000-7f90e32b6000 rw-p 0000c000 ca:01 395727 /lib/x86_64-linux-gnu/libnss_files-2.13.so
        7f90e32b6000-7f90e344b000 r-xp 00000000 ca:01 395724 /lib/x86_64-linux-gnu/libc-2.13.so
        7f90e344b000-7f90e364a000 ---p 00195000 ca:01 395724 /lib/x86_64-linux-gnu/libc-2.13.so
        7f90e364a000-7f90e364e000 r--p 00194000 ca:01 395724 /lib/x86_64-linux-gnu/libc-2.13.so
        7f90e364e000-7f90e364f000 rw-p 00198000 ca:01 395724 /lib/x86_64-linux-gnu/libc-2.13.so
        7f90e364f000-7f90e3655000 rw-p 00000000 00:00 0
        7f90e3655000-7f90e366a000 r-xp 00000000 ca:01 395714 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7f90e366a000-7f90e3869000 ---p 00015000 ca:01 395714 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7f90e3869000-7f90e386a000 r--p 00014000 ca:01 395714 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7f90e386a000-7f90e386b000 rw-p 00015000 ca:01 395714 /lib/x86_64-linux-gnu/libgcc_s.so.1
        7f90e386b000-7f90e38ee000 r-xp 00000000 ca:01 395734 /lib/x86_64-linux-gnu/libm-2.13.so
        7f90e38ee000-7f90e3aed000 ---p 00083000 ca:01 395734 /lib/x86_64-linux-gnu/libm-2.13.so
        7f90e3aed000-7f90e3aee000 r--p 00082000 ca:01 395734 /lib/x86_64-linux-gnu/libm-2.13.so
        7f90e3aee000-7f90e3aef000 rw-p 00083000 ca:01 395734 /lib/x86_64-linux-gnu/libm-2.13.so
        7f90e3aef000-7f90e3bd7000 r-xp 00000000 ca:01 10571 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
        7f90e3bd7000-7f90e3dd7000 ---p 000e8000 ca:01 10571 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
        7f90e3dd7000-7f90e3ddf000 r--p 000e8000 ca:01 10571 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
        7f90e3ddf000-7f90e3de1000 rw-p 000f0000 ca:01 10571 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
        7f90e3de1000-7f90e3df6000 rw-p 00000000 00:00 0
        7f90e3df6000-7f90e3e0e000 r-xp 00000000 ca:01 395732 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7f90e3e0e000-7f90e400d000 ---p 00018000 ca:01 395732 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7f90e400d000-7f90e400e000 r--p 00017000 ca:01 395732 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7f90e400e000-7f90e400f000 rw-p 00018000 ca:01 395732 /lib/x86_64-linux-gnu/libpthread-2.13.so
        7f90e400f000-7f90e4013000 rw-p 00000000 00:00 0
        7f90e4013000-7f90e4034000 r-xp 00000000 ca:01 395731 /lib/x86_64-linux-gnu/ld-2.13.so
        7f90e4125000-7f90e4126000 ---p 00000000 00:00 0
        7f90e4126000-7f90e422b000 rw-p 00000000 00:00 0
        7f90e4230000-7f90e4233000 rw-p 00000000 00:00 0
        7f90e4233000-7f90e4234000 r--p 00020000 ca:01 395731 /lib/x86_64-linux-gnu/ld-2.13.so
        7f90e4234000-7f90e4236000 rw-p 00021000 ca:01 395731 /lib/x86_64-linux-gnu/ld-2.13.so
        7fff9951f000-7fff99540000 rw-p 00000000 00:00 0 [stack]
        7fff995ff000-7fff99600000 r-xp 00000000 00:00 0 [vdso]
        ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
        Received signal 6
        Backtrace: 0x54de95 0x54de95 0x7f90e32ec420 0x7f90e32ec420 0x7f90e32ec3a5 0x7f90e32efb0b 0x7f90e32efb0b 0x7f90e3324113 0x7f90e3324113 0x7f90e3b9352c 0x7f90e332ea96 0x550666 0x7f90e3332d7c 0x7e2295 0x5256c9 0x77fdf9 0x527684 0x793175 0x804350 0x76cf2b 0x7f90e3dfdefc 0x7b4927 0x7f90e339789d 0x7c6cf1
        0x5e6a07 0x7f90e3dfdefc 0x7f90e339789d
        bin/mongos(_ZN5mongo17printStackAndExitEi+0xbin/mongos75()_ZN5mongo17printStackAndExitEi[0x+0x54de9575]
        )[0x54de95]
        /lib/x86_64-linux-gnu/libc.so.6(+0x36420)[0x7f90e32ec420]
        /lib/x86_64-linux-gnu/libc.so.6(+0x36420)[0x7f90e32ec420]
        /lib/x86_64-linux-gnu/libc.so.6(gsignal+0x35)[0x7f90e32ec3a5]
        /lib/x86_64-linux-gnu/libc.so.6(+0x1388e6)[0x7f90e33ee8e6]
        /lib/x86_64-linux-gnu/libc.so.6(abort+0x17b)[0x7f90e32efb0b]
        /usr/lib/x86_64-linux-gnu/libstdc++.so.6(_ZNSs4_Rep8_M_cloneERKSaIcEm+0x72)[0x7f90e3b92e12]
        /lib/x86_64-linux-gnu/libc.so.6(+0x6e113)[0x7f90e3324113]
        /usr/lib/x86_64-linux-gnu/libstdc++.so.6(_ZNSsC1ERKSs+0x3c)[0x7f90e3b9352c]
        /lib/x86_64-linux-gnu/libc.so.6(+0x78a96)[0x7f90e332ea96]
        bin/mongos(_ZN5mongo17getHostNameCachedEv+0x26)[0x550666]
        /lib/x86_64-linux-gnu/libc.so.6(cfree+0x6c)[0x7f90e3332d7c]
        bin/mongos(_ZN5mongo14prettyHostNameEv+0x55)[0x7e2295]
        bin/mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0x1b9)[0x5256c9]
        bin/mongos(_ZN5mongo11dbgrid_cmds15ServerStatusCmd3runERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x29)[0x77fdf9]
        bin/mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runEv+0x74)[0x527684]
        bin/mongos(_ZN5mongo7Command20runAgainstRegisteredEPKcRNS_7BSONObjERNS_14BSONObjBuilderEi+0x8b5)[0x793175]
        bin/mongos(thread_proxy+0x80)[0x804350]
        /lib/x86_64-linux-gnu/libpthread.so.0bin/mongos((+0x_ZN5mongo14SingleStrategy7queryOpERNS_7RequestE7efc+0x)5cb[0x)7f90e3dfdefc[0x]
        76cf2b]
        /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f90e339789d]
        ===
        bin/mongos(_ZN5mongo7Request7processEi+0x187)[0x7b4927]
        bin/mongos(_ZN5mongo21ShardedMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x71)[0x7c6cf1]
        bin/mongos(_ZN5mongo3pms9threadRunEPNS_13MessagingPortE+0x287)[0x5e6a07]
        /lib/x86_64-linux-gnu/libpthread.so.0(+0x7efc)[0x7f90e3dfdefc]
        /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f90e339789d]
        ===
        Logstream::get called in uninitialized state
        Sun Feb 5 04:26:50 [Balancer] reconnect mongo3.foobar.com:27019 failed couldn't connect to server mongo3.foobar.com:27019
        Logstream::get called in uninitialized state
        Sun Feb 5 04:26:50 [Balancer] DBClientCursor::init call() failed
        Logstream::get called in uninitialized state
        Sun Feb 5 04:26:50 [Balancer] query failed to: mongo1.foobar.com:27019 no data
        Logstream::get called in uninitialized state
        Sun Feb 5 04:26:50 [Balancer] Assertion failure ! inShutdown() client/connpool.cpp 136
        0x52b776 0x53662b 0x561a68 0x561bdb 0x562272 0x754b61 0x7d34c4 0x7da611 0x5255cf 0x527684 0x804350 0x7f90e3dfdefc 0x7f90e339789d
        bin/mongos(_ZN5mongo12sayDbContextEPKc+0x96) [0x52b776]
        bin/mongos(_ZN5mongo8assertedEPKcS1_j+0xfb) [0x53662b]
        bin/mongos(_ZN5mongo16DBConnectionPool4_getERKSsd+0x118) [0x561a68]
        bin/mongos(_ZN5mongo16DBConnectionPool3getERKSsd+0x3b) [0x561bdb]
        bin/mongos(_ZN5mongo18ScopedDbConnectionC1EPKNS_5ShardEd+0x92) [0x562272]
        bin/mongos(_ZNK5mongo5Shard10runCommandERKSsRKNS_7BSONObjE+0x31) [0x754b61]
        bin/mongos(_ZN5mongo8Balancer10_checkOIDsEv+0x224) [0x7d34c4]
        bin/mongos(_ZN5mongo8Balancer3runEv+0x331) [0x7da611]
        bin/mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0xbf) [0x5255cf]
        bin/mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runEv+0x74) [0x527684]
        bin/mongos(thread_proxy+0x80) [0x804350]
        /lib/x86_64-linux-gnu/libpthread.so.0(+0x7efc) [0x7f90e3dfdefc]
        /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f90e339789d]
        Logstream::get called in uninitialized state
        Sun Feb 5 04:26:50 [Balancer] ~ScopedDbConnection: _conn != null
        Logstream::get called in uninitialized state
        Logstream::get called in uninitialized state
        Sun Feb 5 04:27:08 [LockPinger] SyncClusterConnection connect fail to: mongo3.foobar.com:27019 errmsg: couldn't connect to server mongo3.foobar.com:27019
        ^CLogstream::get called in uninitialized state
        Sun Feb 5 04:27:11 [CheckConfigServers] dbexit: received signal 2 rc:0 received signal 2

^CLogstream::get called in uninitialized state
Sun Feb 5 04:27:13 [Balancer] dbexit: received signal 2 rc:0 received signal 2

Comment by Mathias Stearn [ 31/Jan/12 ]

Do either of you have a full log showing this error?

Comment by Mathias Stearn [ 18/Jan/12 ]

My bad, that commit was meant for SERVER-4357, not this case.

Comment by Christian Tonhäuser [ 13/Jan/12 ]

We also see this error happening very frequently on our DEV environments since upgrading to 2.0.2

Since that fix has been in the GIT repository for quite a while now, when do you expect it to be integrated in a release?

Here's a backtrace of such a crash on our env:
(Please note: The machine running the crashing mongos process is a XEN-based VM, while the rest of the cluster (6 machines) are "real metal" hardware.)

*** glibc detected *** /data/project/mongodb/bin/mongos: free(): invalid pointer: 0x0000000000b15360 ***
======= Backtrace: =========
/lib64/libc.so.6[0x3933e75676]
/data/project/mongodb/bin/mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0x1b9)[0x5256c9]
/data/project/mongodb/bin/mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runE
v+0x74)[0x527684]
/data/project/mongodb/bin/mongos(thread_proxy+0x80)[0x804350]
/lib64/libpthread.so.0[0x39342077e1]
/lib64/libc.so.6(clone+0x6d)[0x3933ee18ed]
======= Memory map: ========
00400000-008fb000 r-xp 00000000 fd:00 5997190                            /data/project/mongodb-linux-x86_64-2.0.2/bin/mongos
00afb000-00b12000 rw-p 004fb000 fd:00 5997190                            /data/project/mongodb-linux-x86_64-2.0.2/bin/mongos
00b12000-00b16000 rw-p 00000000 00:00 0 
0265b000-0271e000 rw-p 00000000 00:00 0                                  [heap]
3933600000-393361e000 r-xp 00000000 fd:00 390933                         /lib64/ld-2.12.so
393381e000-393381f000 r--p 0001e000 fd:00 390933                         /lib64/ld-2.12.so
393381f000-3933820000 rw-p 0001f000 fd:00 390933                         /lib64/ld-2.12.so
3933820000-3933821000 rw-p 00000000 00:00 0 
3933e00000-3933f75000 r-xp 00000000 fd:00 390935                         /lib64/libc-2.12.so
3933f75000-3934175000 ---p 00175000 fd:00 390935                         /lib64/libc-2.12.so
3934175000-3934179000 r--p 00175000 fd:00 390935                         /lib64/libc-2.12.so
3934179000-393417a000 rw-p 00179000 fd:00 390935                         /lib64/libc-2.12.so
393417a000-393417f000 rw-p 00000000 00:00 0 
3934200000-3934217000 r-xp 00000000 fd:00 391296                         /lib64/libpthread-2.12.so
3934217000-3934417000 ---p 00017000 fd:00 391296                         /lib64/libpthread-2.12.so
3934417000-3934418000 r--p 00017000 fd:00 391296                         /lib64/libpthread-2.12.so
3934418000-3934419000 rw-p 00018000 fd:00 391296                         /lib64/libpthread-2.12.so
3934419000-393441d000 rw-p 00000000 00:00 0 
3934600000-3934683000 r-xp 00000000 fd:00 390943                         /lib64/libm-2.12.so
3934683000-3934882000 ---p 00083000 fd:00 390943                         /lib64/libm-2.12.so
3934882000-3934883000 r--p 00082000 fd:00 390943                         /lib64/libm-2.12.so
3934883000-3934884000 rw-p 00083000 fd:00 390943                         /lib64/libm-2.12.so
3934a00000-3934a0c000 r-xp 00000000 fd:00 390999                         /lib64/libnss_files-2.12.so
3934a0c000-3934c0b000 ---p 0000c000 fd:00 390999                         /lib64/libnss_files-2.12.so
3934c0b000-3934c0c000 r--p 0000b000 fd:00 390999                         /lib64/libnss_files-2.12.so
3934c0c000-3934c0d000 rw-p 0000c000 fd:00 390999                         /lib64/libnss_files-2.12.so
3935600000-3935616000 r-xp 00000000 fd:00 390949                         /lib64/libresolv-2.12.so
3935616000-3935816000 ---p 00016000 fd:00 390949                         /lib64/libresolv-2.12.so
3935816000-3935817000 r--p 00016000 fd:00 390949                         /lib64/libresolv-2.12.so
3935817000-3935818000 rw-p 00017000 fd:00 390949                         /lib64/libresolv-2.12.so
3935818000-393581a000 rw-p 00000000 00:00 0 
3937a00000-3937a16000 r-xp 00000000 fd:00 391309                         /lib64/libgcc_s-4.4.4-20100726.so.1
3937a16000-3937c15000 ---p 00016000 fd:00 391309                         /lib64/libgcc_s-4.4.4-20100726.so.1
3937c15000-3937c16000 rw-p 00015000 fd:00 391309                         /lib64/libgcc_s-4.4.4-20100726.so.1
393a200000-393a2e9000 r-xp 00000000 fd:00 796275                         /usr/lib64/libstdc++.so.6.0.13
393a2e9000-393a4e9000 ---p 000e9000 fd:00 796275                         /usr/lib64/libstdc++.so.6.0.13
393a4e9000-393a4f0000 r--p 000e9000 fd:00 796275                         /usr/lib64/libstdc++.so.6.0.13
393a4f0000-393a4f2000 rw-p 000f0000 fd:00 796275                         /usr/lib64/libstdc++.so.6.0.13
[snip, removed lots of basically empty lines]
7f41401d1000-7f41401d6000 r-xp 00000000 fd:00 390942                     /lib64/libnss_dns-2.12.so
7f41401d6000-7f41403d5000 ---p 00005000 fd:00 390942                     /lib64/libnss_dns-2.12.so
7f41403d5000-7f41403d6000 r--p 00004000 fd:00 390942                     /lib64/libnss_dns-2.12.so
7f41403d6000-7f41403d7000 rw-p 00005000 fd:00 390942                     /lib64/libnss_dns-2.12.so
7f41403de000-7f41403e3000 rw-p 00000000 00:00 0 
7f41403e9000-7f41403eb000 rw-p 00000000 00:00 0 
7fffb6ff8000-7fffb700d000 rw-p 00000000 00:00 0                          [stack]
7fffb7060000-7fffb7061000 r-xp 00000000 00:00 0                          [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
Received signal 6
Backtrace: 0x54de95 0x3933e32a20 0x3933e329a5 0x3933e34185 0x3933e6fd5b 0x3933e75676 0x5256c9 0x527684 0x804350 0x39342077e1 0x3933ee18ed 
/data/mapwiki/mongodb/bin/mongos(_ZN5mongo17printStackAndExitEi+0x75)[0x54de95]
/lib64/libc.so.6[0x3933e32a20]
/lib64/libc.so.6(gsignal+0x35)[0x3933e329a5]
/lib64/libc.so.6(abort+0x175)[0x3933e34185]
/lib64/libc.so.6[0x3933e6fd5b]
/lib64/libc.so.6[0x3933e75676]
/data/mapwiki/mongodb/bin/mongos(_ZN5mongo13BackgroundJob7jobBodyEN5boost10shared_ptrINS0_9JobStatusEEE+0x1b9)[0x5256c9]
/data/mapwiki/mongodb/bin/mongos(_ZN5boost6detail11thread_dataINS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo13BackgroundJobENS_10shared_ptrINS7_9JobStatusEEEEENS2_5list2INS2_5valueIPS7_EENSD_ISA_EEEEEEE3runEv+0x74)[0x527684]
/data/mapwiki/mongodb/bin/mongos(thread_proxy+0x80)[0x804350]
/lib64/libpthread.so.0[0x39342077e1]
/lib64/libc.so.6(clone+0x6d)[0x3933ee18ed]
===

Comment by auto [ 28/Nov/11 ]

Author:

{u'login': u'RedBeard0531', u'name': u'Mathias Stearn', u'email': u'mathias@10gen.com'}

Message: Explicitly don't allow appending unsigned ints to BSONObjBuilder SERVER-4367
Branch: master
https://github.com/mongodb/mongo/commit/cb41d6515b220a4967331eeb873891dd33f590bb

Generated at Thu Feb 08 03:05:47 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.