Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-23168

Shell killOperationsOnAllConnections doesn't handle currentOp failure

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Minor - P4
    • Resolution: Fixed
    • Affects Version/s: None
    • Fix Version/s: 3.3.11
    • Component/s: Shell
    • Labels:
    • Backwards Compatibility:
      Fully Compatible
    • Operating System:
      ALL
    • Case:

      Description

      When the user interrupts the shell with Ctrl-C (or similar), the shell will check to see if any operations are in progress, and if so, prompt the user to kill them. However, after getting the currentOp results, the code assumes that there will be an inprog field — but this is not present if the currentOp fails (for example, due to a lack of auth). This causes an assertion to be tripped inside BSONElement::embeddedObject().

      304
      void ConnectionRegistry::killOperationsOnAllConnections(bool withPrompt) const {
      305
          Prompter prompter("do you want to kill the current op(s) on the server?");
      306
          stdx::lock_guard<stdx::mutex> lk(_mutex);
      307
          for (map<string, set<string>>::const_iterator i = _connectionUris.begin();
      308
               i != _connectionUris.end();
      309
               ++i) {
      310
              auto status = ConnectionString::parse(i->first);
      311
              if (!status.isOK()) {
      312
                  continue;
      313
              }
      314
       
      315
              const ConnectionString cs(status.getValue());
      316
       
      317
              string errmsg;
      318
              std::unique_ptr<DBClientWithCommands> conn(cs.connect(errmsg));
      319
              if (!conn) {
      320
                  continue;
      321
              }
      322
       
      323
              const set<string>& uris = i->second;
      324
       
      325
              BSONObj currentOpRes;
      326
              conn->runPseudoCommand("admin", "currentOp", "$cmd.sys.inprog", {}, currentOpRes);
      327
              auto inprog = currentOpRes["inprog"].embeddedObject();
      328
              BSONForEach(op, inprog) {
      329
                  if (uris.count(op["client"].String())) {
      330
                      if (!withPrompt || prompter.confirm()) {
      331
                          BSONObjBuilder cmdBob;
      332
                          BSONObj info;
      333
                          cmdBob.append("op", op["opid"]);
      334
                          auto cmdArgs = cmdBob.done();
      335
                          conn->runPseudoCommand("admin", "killOp", "$cmd.sys.killop", cmdArgs, info);
      336
                      } else {
      337
                          return;
      338
                      }
      339
                  }
      340
              }
      341
          }
      342
      }
      

      Repro:

      • Start auth mongod
      • Run a slightly slow non-privileged command
      • Immediately hit Ctrl-C

      $ mlaunch init --binarypath /m/3.2.4/bin --single --auth --port 1111
      launching: /m/3.2.4/bin/mongod on port 1111
      $ /m/3.2.4/bin/mongo --port 1111
      MongoDB shell version: 3.2.4
      connecting to: 127.0.0.1:1111/test
      > db.runCommand({ping:1, payload:(new Array(1048576)).join('x')})
      ^C2016-03-16T19:03:03.458+1100 I -        [main] Assertion failure isABSONObj() src/mongo/bson/bsonelement.cpp 407
      2016-03-16T19:03:03.467+1100 I CONTROL  [main]
       0x961222 0x910248 0x9029c4 0x71b462 0x8eb5d9 0x7002a0 0x7f8c11174c30 0x7f8c11510412 0x1038b1c 0x8bfdeb 0x8c21a0 0x701e7f 0x7064cb 0x7f8c1115fec5 0x6ff859
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"561222","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"510248","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"5029C4","s":"_ZN5mongo12verifyFailedEPKcS1_j"},{"b":"400000","o":"31B462","s":"_ZNK5mongo11BSONElement14embeddedObjectEv"},{"b":"400000","o":"4EB5D9","s":"_ZNK5mongo11shell_utils18ConnectionRegistry30killOperationsOnAllConnectionsEb"},{"b":"400000","o":"3002A0","s":"_ZN5mongo11exitCleanlyENS_8ExitCodeE"},{"b":"7F8C1113E000","o":"36C30"},{"b":"7F8C11504000","o":"C412","s":"pthread_cond_wait"},{"b":"400000","o":"C38B1C","s":"_ZNSt18condition_variable4waitERSt11unique_lockISt5mutexE"},{"b":"400000","o":"4BFDEB","s":"_ZN5mongo5mozjs15MozJSProxyScope15runOnImplThreadESt8functionIFvvEE"},{"b":"400000","o":"4C21A0","s":"_ZN5mongo5mozjs15MozJSProxyScope4execENS_10StringDataERKSsbbbi"},{"b":"400000","o":"301E7F","s":"_Z5_mainiPPcS0_"},{"b":"400000","o":"3064CB","s":"main"},{"b":"7F8C1113E000","o":"21EC5","s":"__libc_start_main"},{"b":"400000","o":"2FF859"}],"processInfo":{ "mongodbVersion" : "3.2.4", "gitVersion" : "e2ee9ffcf9f5a94fad76802e28cc978718bb7a30", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.13.0-39-generic", "version" : "#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFFC70FE000", "elfType" : 3 }, { "b" : "7F8C12146000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7F8C11F42000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7F8C11C3E000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7F8C11938000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7F8C11722000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7F8C11504000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7F8C1113E000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7F8C1234E000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
       mongo(_ZN5mongo15printStackTraceERSo+0x32) [0x961222]
       mongo(_ZN5mongo10logContextEPKc+0x138) [0x910248]
       mongo(_ZN5mongo12verifyFailedEPKcS1_j+0xB4) [0x9029c4]
       mongo(_ZNK5mongo11BSONElement14embeddedObjectEv+0x32) [0x71b462]
       mongo(_ZNK5mongo11shell_utils18ConnectionRegistry30killOperationsOnAllConnectionsEb+0x3D9) [0x8eb5d9]
       mongo(_ZN5mongo11exitCleanlyENS_8ExitCodeE+0x40) [0x7002a0]
       libc.so.6(+0x36C30) [0x7f8c11174c30]
       libpthread.so.0(pthread_cond_wait+0xC2) [0x7f8c11510412]
       mongo(_ZNSt18condition_variable4waitERSt11unique_lockISt5mutexE+0xC) [0x1038b1c]
       mongo(_ZN5mongo5mozjs15MozJSProxyScope15runOnImplThreadESt8functionIFvvEE+0x10B) [0x8bfdeb]
       mongo(_ZN5mongo5mozjs15MozJSProxyScope4execENS_10StringDataERKSsbbbi+0xB0) [0x8c21a0]
       mongo(_Z5_mainiPPcS0_+0xFFF) [0x701e7f]
       mongo(main+0x2B) [0x7064cb]
       libc.so.6(__libc_start_main+0xF5) [0x7f8c1115fec5]
       mongo(+0x2FF859) [0x6ff859]
      -----  END BACKTRACE  -----
      error:assertion src/mongo/bson/bsonelement.cpp:407
      2016-03-16T19:03:03.497+1100 I -        [main] Invariant failure _state == State::Idle src/mongo/scripting/mozjs/proxyscope.cpp 275
      2016-03-16T19:03:03.497+1100 I -        [main]
       
      ***aborting after invariant() failure
       
       
      2016-03-16T19:03:03.500+1100 F -        [main] Got signal: 6 (Aborted).
       
       0x961222 0x960149 0x960952 0x7f8c11514340 0x7f8c11174bb9 0x7f8c11177fc8 0x901fab 0x8bff51 0x8c1d41 0x7017cf 0x7064cb 0x7f8c1115fec5 0x6ff859
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"561222","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"560149"},{"b":"400000","o":"560952"},{"b":"7F8C11504000","o":"10340"},{"b":"7F8C1113E000","o":"36BB9","s":"gsignal"},{"b":"7F8C1113E000","o":"39FC8","s":"abort"},{"b":"400000","o":"501FAB","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"4BFF51","s":"_ZN5mongo5mozjs15MozJSProxyScope15runOnImplThreadESt8functionIFvvEE"},{"b":"400000","o":"4C1D41","s":"_ZN5mongo5mozjs15MozJSProxyScope4typeEPKc"},{"b":"400000","o":"3017CF","s":"_Z5_mainiPPcS0_"},{"b":"400000","o":"3064CB","s":"main"},{"b":"7F8C1113E000","o":"21EC5","s":"__libc_start_main"},{"b":"400000","o":"2FF859"}],"processInfo":{ "mongodbVersion" : "3.2.4", "gitVersion" : "e2ee9ffcf9f5a94fad76802e28cc978718bb7a30", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.13.0-39-generic", "version" : "#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFFC70FE000", "elfType" : 3 }, { "b" : "7F8C12146000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7F8C11F42000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7F8C11C3E000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7F8C11938000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7F8C11722000", "path"
      : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7F8C11504000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7F8C1113E000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7F8C1234E000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
       mongo(_ZN5mongo15printStackTraceERSo+0x32) [0x961222]
       mongo(+0x560149) [0x960149]
       mongo(+0x560952) [0x960952]
       libpthread.so.0(+0x10340) [0x7f8c11514340]
       libc.so.6(gsignal+0x39) [0x7f8c11174bb9]
       libc.so.6(abort+0x148) [0x7f8c11177fc8]
       mongo(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x901fab]
       mongo(_ZN5mongo5mozjs15MozJSProxyScope15runOnImplThreadESt8functionIFvvEE+0x271) [0x8bff51]
       mongo(_ZN5mongo5mozjs15MozJSProxyScope4typeEPKc+0x71) [0x8c1d41]
       mongo(_Z5_mainiPPcS0_+0x94F) [0x7017cf]
       mongo(main+0x2B) [0x7064cb]
       libc.so.6(__libc_start_main+0xF5) [0x7f8c1115fec5]
       mongo(+0x2FF859) [0x6ff859]
      -----  END BACKTRACE  -----
      Aborted (core dumped)
      

        Attachments

          Issue Links

            Activity

              People

              Assignee:
              nathan.typanski Nathan Typanski
              Reporter:
              kevin.pulo Kevin Pulo
              Participants:
              Votes:
              0 Vote for this issue
              Watchers:
              8 Start watching this issue

                Dates

                Created:
                Updated:
                Resolved: