Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-32830

MongoDB Server - Slow query performance problem

    XMLWordPrintableJSON

Details

    • Icon: Question Question
    • Resolution: Done
    • Icon: Major - P3 Major - P3
    • None
    • None
    • None
    • None

    Description

      Hello,
      I have a problem with my MongoDB cluster: many queries appear to be queued, slowing the system down.
      My MongoDB is composed of three server in replicaset.

      Server (Primary) TOP:

      top - 15:24:11 up 139 days, 22:14,  6 users,  load average: 99,04, 98,94, 98,86
      Tasks: 143 total,   5 running, 138 sleeping,   0 stopped,   0 zombie
      %Cpu(s):  6,4 us, 29,4 sy,  0,0 ni,  0,0 id, 57,3 wa,  0,0 hi,  6,9 si,  0,0 st
      KiB Mem :  4817956 total,    41648 free,  3246844 used,  1529464 buff/cache
      KiB Swap:  8388604 total,  8324416 free,    64188 used.  1273813 avail Mem 
       
        PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND                             
      24862 mongod    20   0 27,789g 3,001g 209216 R  54,8 65,3 132:12.61 mongod  
      

      db.serverStatus():

      { 
          "host" : "mongodbsrv01", 
          "version" : "3.0.8", 
          "process" : "mongod", 
          "pid" : NumberLong(24862), 
          "uptime" : 8267.0, 
          "uptimeMillis" : NumberLong(8266883), 
          "uptimeEstimate" : 6982.0, 
          "localTime" : ISODate("2018-01-22T14:24:27.341+0000"), 
          "asserts" : {
              "regular" : 0.0, 
              "warning" : 0.0, 
              "msg" : 0.0, 
              "user" : 53.0, 
              "rollovers" : 0.0
          }, 
          "backgroundFlushing" : {
              "flushes" : 137.0, 
              "total_ms" : 10382.0, 
              "average_ms" : 75.78102189781022, 
              "last_ms" : 0.0, 
              "last_finished" : ISODate("2018-01-22T14:23:40.786+0000")
          }, 
          "connections" : {
              "current" : 123.0, 
              "available" : 51077.0, 
              "totalCreated" : NumberLong(994)
          }, 
          "cursors" : {
              "note" : "deprecated, use server status metrics", 
              "clientCursors_size" : 76.0, 
              "totalOpen" : 76.0, 
              "pinned" : 74.0, 
              "totalNoTimeout" : 74.0, 
              "timedOut" : 1.0
          }, 
          "dur" : {
              "commits" : 29.0, 
              "journaledMB" : 0.0, 
              "writeToDataFilesMB" : 0.0, 
              "compression" : 0.0, 
              "commitsInWriteLock" : 0.0, 
              "earlyCommits" : 0.0, 
              "timeMs" : {
                  "dt" : 3003.0, 
                  "prepLogBuffer" : 0.0, 
                  "writeToJournal" : 0.0, 
                  "writeToDataFiles" : 0.0, 
                  "remapPrivateView" : 0.0, 
                  "commits" : 1.0, 
                  "commitsInWriteLock" : 0.0
              }
          }, 
          "extra_info" : {
              "note" : "fields vary by platform", 
              "heap_usage_bytes" : 443257136.0, 
              "page_faults" : 2302153.0
          }, 
          "globalLock" : {
              "totalTime" : NumberLong(8267149000), 
              "currentQueue" : {
                  "total" : 0.0, 
                  "readers" : 0.0, 
                  "writers" : 0.0
              }, 
              "activeClients" : {
                  "total" : 139.0, 
                  "readers" : 96.0, 
                  "writers" : 0.0
              }
          }, 
          "locks" : {
              "Global" : {
                  "acquireCount" : {
                      "r" : NumberLong(6225014), 
                      "w" : NumberLong(4566), 
                      "R" : NumberLong(20), 
                      "W" : NumberLong(25)
                  }, 
                  "acquireWaitCount" : {
                      "r" : NumberLong(1)
                  }, 
                  "timeAcquiringMicros" : {
                      "r" : NumberLong(176)
                  }
              }, 
              "MMAPV1Journal" : {
                  "acquireCount" : {
                      "r" : NumberLong(3125940), 
                      "w" : NumberLong(26144567), 
                      "R" : NumberLong(72508), 
                      "W" : NumberLong(77)
                  }, 
                  "acquireWaitCount" : {
                      "r" : NumberLong(2058), 
                      "w" : NumberLong(2011), 
                      "R" : NumberLong(1970), 
                      "W" : NumberLong(20)
                  }, 
                  "timeAcquiringMicros" : {
                      "r" : NumberLong(494344822), 
                      "w" : NumberLong(8148242), 
                      "R" : NumberLong(186276693), 
                      "W" : NumberLong(4214859)
                  }
              }, 
              "Database" : {
                  "acquireCount" : {
                      "r" : NumberLong(3110195), 
                      "w" : NumberLong(4501), 
                      "R" : NumberLong(27), 
                      "W" : NumberLong(65)
                  }, 
                  "acquireWaitCount" : {
                      "r" : NumberLong(123), 
                      "w" : NumberLong(1), 
                      "R" : NumberLong(1), 
                      "W" : NumberLong(2)
                  }, 
                  "timeAcquiringMicros" : {
                      "r" : NumberLong(37857023446), 
                      "w" : NumberLong(493870), 
                      "R" : NumberLong(8302), 
                      "W" : NumberLong(259911)
                  }
              }, 
              "Collection" : {
                  "acquireCount" : {
                      "R" : NumberLong(3091273), 
                      "W" : NumberLong(4110)
                  }, 
                  "acquireWaitCount" : {
                      "R" : NumberLong(15317), 
                      "W" : NumberLong(164)
                  }, 
                  "timeAcquiringMicros" : {
                      "R" : NumberLong(7360200399), 
                      "W" : NumberLong(31201699)
                  }
              }, 
              "Metadata" : {
                  "acquireCount" : {
                      "R" : NumberLong(4), 
                      "W" : NumberLong(367)
                  }
              }, 
              "oplog" : {
                  "acquireCount" : {
                      "w" : NumberLong(391), 
                      "R" : NumberLong(21173)
                  }, 
                  "acquireWaitCount" : {
                      "w" : NumberLong(2), 
                      "R" : NumberLong(262)
                  }, 
                  "timeAcquiringMicros" : {
                      "w" : NumberLong(255158), 
                      "R" : NumberLong(10547879)
                  }
              }
          }, 
          "network" : {
              "bytesIn" : NumberLong(40962593), 
              "bytesOut" : NumberLong(504146686), 
              "numRequests" : NumberLong(37039)
          }, 
          "opcounters" : {
              "insert" : 120.0, 
              "query" : 5784.0, 
              "update" : 0.0, 
              "delete" : 0.0, 
              "getmore" : 4130.0, 
              "command" : 27134.0
          }, 
          "opcountersRepl" : {
              "insert" : 4.0, 
              "query" : 0.0, 
              "update" : 18.0, 
              "delete" : 0.0, 
              "getmore" : 0.0, 
              "command" : 0.0
          }, 
          "repl" : {
              "setName" : "docuReplicaSet", 
              "setVersion" : 7.0, 
              "ismaster" : true, 
              "secondary" : false, 
              "hosts" : [
                  "mongodbsrv01:27017", 
                  "mongodbsrv02:27017", 
                  "mongodbsrv03:27017"
              ], 
              "primary" : "mongodbsrv01:27017", 
              "me" : "mongodbsrv01:27017", 
              "electionId" : ObjectId("5a65d4eac297c29a38ecb5b2"), 
              "rbid" : 452501248.0
          }, 
          "storageEngine" : {
              "name" : "mmapv1"
          }, 
          "writeBacksQueued" : false, 
          "mem" : {
              "bits" : 64.0, 
              "resident" : 3096.0, 
              "virtual" : 28455.0, 
              "supported" : true, 
              "mapped" : 12395.0, 
              "mappedWithJournal" : 24790.0
          }, 
          "metrics" : {
              "commands" : {
                  "<UNKNOWN>" : NumberLong(3), 
                  "_isSelf" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(2)
                  }, 
                  "aggregate" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(640)
                  }, 
                  "buildInfo" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(298)
                  }, 
                  "collStats" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(3)
                  }, 
                  "connectionStatus" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(15)
                  }, 
                  "count" : {
                      "failed" : NumberLong(91), 
                      "total" : NumberLong(10032)
                  }, 
                  "dbStats" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(4)
                  }, 
                  "findAndModify" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(271)
                  }, 
                  "getLastError" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(292)
                  }, 
                  "insert" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(120)
                  }, 
                  "isMaster" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(3803)
                  }, 
                  "listCollections" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(19)
                  }, 
                  "listDatabases" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(1)
                  }, 
                  "planCacheClear" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(2)
                  }, 
                  "planCacheListQueryShapes" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(1)
                  }, 
                  "reIndex" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(1)
                  }, 
                  "replSetElect" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(1)
                  }, 
                  "replSetFresh" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(4)
                  }, 
                  "replSetGetStatus" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(133)
                  }, 
                  "replSetHeartbeat" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(8226)
                  }, 
                  "replSetUpdatePosition" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(790)
                  }, 
                  "saslContinue" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(1716)
                  }, 
                  "saslStart" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(858)
                  }, 
                  "serverStatus" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(20)
                  }, 
                  "usersInfo" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(1)
                  }, 
                  "whatsmyuri" : {
                      "failed" : NumberLong(0), 
                      "total" : NumberLong(3)
                  }
              }, 
              "cursor" : {
                  "timedOut" : NumberLong(1), 
                  "open" : {
                      "noTimeout" : NumberLong(74), 
                      "pinned" : NumberLong(74), 
                      "total" : NumberLong(76)
                  }
              }, 
              "document" : {
                  "deleted" : NumberLong(0), 
                  "inserted" : NumberLong(120), 
                  "returned" : NumberLong(1221273), 
                  "updated" : NumberLong(271)
              }, 
              "getLastError" : {
                  "wtime" : {
                      "num" : 0.0, 
                      "totalMillis" : 0.0
                  }, 
                  "wtimeouts" : NumberLong(0)
              }, 
              "operation" : {
                  "fastmod" : NumberLong(234), 
                  "idhack" : NumberLong(0), 
                  "scanAndOrder" : NumberLong(10), 
                  "writeConflicts" : NumberLong(0)
              }, 
              "queryExecutor" : {
                  "scanned" : NumberLong(558924), 
                  "scannedObjects" : NumberLong(530425)
              }, 
              "record" : {
                  "moves" : NumberLong(13)
              }, 
              "repl" : {
                  "apply" : {
                      "batches" : {
                          "num" : 19.0, 
                          "totalMillis" : 974.0
                      }, 
                      "ops" : NumberLong(22)
                  }, 
                  "buffer" : {
                      "count" : NumberLong(0), 
                      "maxSizeBytes" : 268435456.0, 
                      "sizeBytes" : NumberLong(0)
                  }, 
                  "network" : {
                      "bytes" : NumberLong(54827), 
                      "getmores" : {
                          "num" : 67.0, 
                          "totalMillis" : 280404.0
                      }, 
                      "ops" : NumberLong(22), 
                      "readersCreated" : NumberLong(1)
                  }, 
                  "preload" : {
                      "docs" : {
                          "num" : 18.0, 
                          "totalMillis" : 0.0
                      }, 
                      "indexes" : {
                          "num" : 511.0, 
                          "totalMillis" : 350.0
                      }
                  }
              }, 
              "storage" : {
                  "freelist" : {
                      "search" : {
                          "bucketExhausted" : NumberLong(0), 
                          "requests" : NumberLong(163359), 
                          "scanned" : NumberLong(0)
                      }
                  }
              }, 
              "ttl" : {
                  "deletedDocuments" : NumberLong(0), 
                  "passes" : NumberLong(124)
              }
          }, 
          "ok" : 1.0
      }
      

      Can anybody help me?

      Attachments

        Activity

          People

            kelsey.schubert@mongodb.com Kelsey Schubert
            biagio.tozzi Biagio Tozzi
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: