Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-3205

Dropping sharded database leads to errors with dbnames which are prefixes of dropped name

    • Type: Icon: Bug Bug
    • Resolution: Duplicate
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: None
    • Labels:
      None
    • ALL

      Hi,

      There is a issue which is causing mongos to crash. Following is the description:

      • 2 shards configured with one router and one config server.
      • Using mongo shell to configure shards. Configured sharding on 2 databases named "tutorial1" and "tutorial10"
      • Insert a document in "tutorial1"
      • Drop database "tutorial1"
      • Insert a document in "tutorial10"
        mongos will crash and an exception is received at mongo shell

      Following are the set of commands to configure above setup and replicate crash.

      Run mongo instances
      ./mongod --bind_ip 192.168.50.68 --port 20000 --shardsvr --dbpath /sata1/master
      ./mongod --bind_ip 192.168.50.68 --port 25000 --shardsvr --dbpath /sata2/master
      ./mongod --bind_ip 192.168.50.68 --port 30000 --configsvr --dbpath /sata1/configdb
      ./mongos --bind_ip 192.168.50.68 --port 35000 --configdb 192.168.50.68:30000

      Configure using mongo shell
      use admin
      db.admin.runCommand(

      {"addshard" : "192.168.50.68:20000"}

      );
      db.admin.runCommand(

      {"addshard" : "192.168.50.68:25000"}

      );

      db.runCommand(

      { enablesharding : "tutorial1" }

      )
      db.runCommand( { shardcollection : "tutorial1.testsh", key : {_sk : 1} } )

      db.runCommand(

      { enablesharding : "tutorial10" }

      )
      db.runCommand( { shardcollection : "tutorial10.testsh", key : {_sk : 1} } )

      Crash Replication commands
      use tutorial1
      db.testsh.save({_id:"id_1",_sk:"1"})
      db.dropDatabase()

      use tutorial10
      db.testsh.save({_id:"id_1",_sk:"1"}) //This will Cause Crash

      Error message visible on mongo shell
      Thu Mar 11 06:51:39 MessagingPort recv() errno:104 Connection reset by peer 192.168.50.68:35000
      Thu Mar 11 06:51:39 SocketException: remote: error: 9001 socket exception [1]
      Thu Mar 11 06:51:39 DBClientCursor::init call() failed
      Thu Mar 11 06:51:39 query failed : tutorial10.$cmd

      { getlasterror: 1.0, w: 1.0 }

      to: 192.168.50.68:35000
      Thu Mar 11 06:51:39 Error: error doing query: failed shell/collection.js:150
      Thu Mar 11 06:51:39 trying reconnect to 192.168.50.68:35000
      Thu Mar 11 06:51:39 reconnect 192.168.50.68:35000 failed couldn't connect to server 192.168.50.68:35000

      Crash Log found at mongos
      mongos: /opt/extra/include/boost/smart_ptr/shared_ptr.hpp:418: T* boost::shared_ptr< <template-parameter-1-1> >::operator->() const [with T = mongo::ChunkManager]: Assertion `px != 0' failed.
      Received signal 6
      Backtrace: 0x52e235 0x3e19e302d0 0x3e19e30265 0x3e19e31d10 0x3e19e296e6 0x69b2dd 0x69acf2 0x576ba6 0x5774b6 0x575630 0x575b31 0x62dcdd 0x63913d 0x63a1e3 0x664202 0x6761c7 0x57ea3c 0x69ec30 0x3e1aa0673d 0x3e19ed3d1d
      ./mongos(_ZN5mongo17printStackAndExitEi+0x75)[0x52e235]
      /lib64/libc.so.6[0x3e19e302d0]
      /lib64/libc.so.6(gsignal+0x35)[0x3e19e30265]
      /lib64/libc.so.6(abort+0x110)[0x3e19e31d10]
      /lib64/libc.so.6(__assert_fail+0xf6)[0x3e19e296e6]
      ./mongos[0x69b2dd]
      ./mongos[0x69acf2]
      ./mongos(_ZN5boost6detail8function17function_invoker4IPFbRN5mongo12DBClientBaseERKSsbiEbS5_S7_biE6invokeERNS1_15function_bufferES5_S7_bi+0x16)[0x576ba6]
      ./mongos(_ZN5mongo17ClientConnections13checkVersionsERKSs+0x1c6)[0x5774b6]
      ./mongos(_ZN5mongo15ShardConnection5_initEv+0x2d0)[0x575630]
      ./mongos(_ZN5mongo15ShardConnectionC1ERKNS_5ShardERKSs+0xa1)[0x575b31]
      ./mongos(_ZN5mongo8Strategy7doWriteEiRNS_7RequestERKNS_5ShardEb+0x4d)[0x62dcdd]
      ./mongos(_ZN5mongo13ShardStrategy7_updateERNS_7RequestERNS_9DbMessageEN5boost10shared_ptrINS_12ChunkManagerEEE+0xd8d)[0x63913d]
      ./mongos(_ZN5mongo13ShardStrategy7writeOpEiRNS_7RequestE+0x2b3)[0x63a1e3]
      ./mongos(_ZN5mongo7Request7processEi+0x172)[0x664202]
      ./mongos(_ZN5mongo21ShardedMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x77)[0x6761c7]
      ./mongos(_ZN5mongo3pms9threadRunEPNS_13MessagingPortE+0x34c)[0x57ea3c]
      ./mongos(thread_proxy+0x80)[0x69ec30]
      /lib64/libpthread.so.0[0x3e1aa0673d]
      /lib64/libc.so.6(clone+0x6d)[0x3e19ed3d1d]
      ===
      Received signal 11
      Backtrace: 0x52e235 0x3e19e302d0 0x3e19e7c126 0x3e2c49c200 0x3e2c49c8ff 0x532ed0 0x577ffc 0x5789dc 0x69f3c1 0x573b22 0x3e19e333a5 0x52e29b 0x3e19e302d0 0x3e19e30265 0x3e19e31d10 0x3e19e296e6 0x69b2dd 0x69acf2 0x576ba6 0x5774b6
      ./mongos(_ZN5mongo17printStackAndExitEi+0x75)[0x52e235]
      /lib64/libc.so.6[0x3e19e302d0]
      /lib64/libc.so.6(memcpy+0x286)[0x3e19e7c126]
      /usr/lib64/libstdc++.so.6(_ZNSs4_Rep8_M_cloneERKSaIcEm+0x50)[0x3e2c49c200]
      /usr/lib64/libstdc++.so.6(_ZNSsC1ERKSs+0x4f)[0x3e2c49c8ff]
      ./mongos(ZNK5mongo16DBConnectionPool17serverNameCompareclERKSsS3+0x170)[0x532ed0]
      ./mongos(_ZN5mongo17ClientConnections7releaseERKSsPNS_12DBClientBaseE+0x25c)[0x577ffc]
      ./mongos(_ZN5boost19thread_specific_ptrIN5mongo17ClientConnectionsEE11delete_dataclEPv+0xac)[0x5789dc]
      ./mongos(_ZN5boost6detail12set_tss_dataEPKvNS_10shared_ptrINS0_20tss_cleanup_functionEEEPvb+0x151)[0x69f3c1]
      ./mongos[0x573b22]
      /lib64/libc.so.6(exit+0xe5)[0x3e19e333a5]
      ./mongos[0x52e29b]
      /lib64/libc.so.6[0x3e19e302d0]
      /lib64/libc.so.6(gsignal+0x35)[0x3e19e30265]
      /lib64/libc.so.6(abort+0x110)[0x3e19e31d10]
      /lib64/libc.so.6(__assert_fail+0xf6)[0x3e19e296e6]
      ./mongos[0x69b2dd]
      ./mongos[0x69acf2]
      ./mongos(_ZN5boost6detail8function17function_invoker4IPFbRN5mongo12DBClientBaseERKSsbiEbS5_S7_biE6invokeERNS1_15function_bufferES5_S7_bi+0x16)[0x576ba6]
      ./mongos(_ZN5mongo17ClientConnections13checkVersionsERKSs+0x1c6)[0x5774b6]
      ===
      [root@localhost mongobin_1.8.1]#

      Note:
      After more digging, it has been observed that since "tutorial1" database name is a sub-string of another database "tutorial10". And "tutorial1" is dropped, then any operation on "tutorial10" (or any database name starting with tutorial1) will cause crash of mongos.

      Please suggest. Thanks!

      Regards,
      Nikhil

            Assignee:
            Unassigned Unassigned
            Reporter:
            greg_10gen Greg Studer
            Votes:
            0 Vote for this issue
            Watchers:
            0 Start watching this issue

              Created:
              Updated:
              Resolved: