Details
-
Question
-
Resolution: Done
-
Major - P3
-
None
-
None
-
None
-
None
Description
Hello,
I have a problem with my MongoDB cluster: many queries appear to be queued, slowing the system down.
My MongoDB is composed of three server in replicaset.
Server (Primary) TOP:
top - 15:24:11 up 139 days, 22:14, 6 users, load average: 99,04, 98,94, 98,86
|
Tasks: 143 total, 5 running, 138 sleeping, 0 stopped, 0 zombie
|
%Cpu(s): 6,4 us, 29,4 sy, 0,0 ni, 0,0 id, 57,3 wa, 0,0 hi, 6,9 si, 0,0 st
|
KiB Mem : 4817956 total, 41648 free, 3246844 used, 1529464 buff/cache
|
KiB Swap: 8388604 total, 8324416 free, 64188 used. 1273813 avail Mem
|
|
|
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
|
24862 mongod 20 0 27,789g 3,001g 209216 R 54,8 65,3 132:12.61 mongod
|
db.serverStatus():
{
|
"host" : "mongodbsrv01",
|
"version" : "3.0.8",
|
"process" : "mongod",
|
"pid" : NumberLong(24862),
|
"uptime" : 8267.0,
|
"uptimeMillis" : NumberLong(8266883),
|
"uptimeEstimate" : 6982.0,
|
"localTime" : ISODate("2018-01-22T14:24:27.341+0000"),
|
"asserts" : {
|
"regular" : 0.0,
|
"warning" : 0.0,
|
"msg" : 0.0,
|
"user" : 53.0,
|
"rollovers" : 0.0
|
},
|
"backgroundFlushing" : {
|
"flushes" : 137.0,
|
"total_ms" : 10382.0,
|
"average_ms" : 75.78102189781022,
|
"last_ms" : 0.0,
|
"last_finished" : ISODate("2018-01-22T14:23:40.786+0000")
|
},
|
"connections" : {
|
"current" : 123.0,
|
"available" : 51077.0,
|
"totalCreated" : NumberLong(994)
|
},
|
"cursors" : {
|
"note" : "deprecated, use server status metrics",
|
"clientCursors_size" : 76.0,
|
"totalOpen" : 76.0,
|
"pinned" : 74.0,
|
"totalNoTimeout" : 74.0,
|
"timedOut" : 1.0
|
},
|
"dur" : {
|
"commits" : 29.0,
|
"journaledMB" : 0.0,
|
"writeToDataFilesMB" : 0.0,
|
"compression" : 0.0,
|
"commitsInWriteLock" : 0.0,
|
"earlyCommits" : 0.0,
|
"timeMs" : {
|
"dt" : 3003.0,
|
"prepLogBuffer" : 0.0,
|
"writeToJournal" : 0.0,
|
"writeToDataFiles" : 0.0,
|
"remapPrivateView" : 0.0,
|
"commits" : 1.0,
|
"commitsInWriteLock" : 0.0
|
}
|
},
|
"extra_info" : {
|
"note" : "fields vary by platform",
|
"heap_usage_bytes" : 443257136.0,
|
"page_faults" : 2302153.0
|
},
|
"globalLock" : {
|
"totalTime" : NumberLong(8267149000),
|
"currentQueue" : {
|
"total" : 0.0,
|
"readers" : 0.0,
|
"writers" : 0.0
|
},
|
"activeClients" : {
|
"total" : 139.0,
|
"readers" : 96.0,
|
"writers" : 0.0
|
}
|
},
|
"locks" : {
|
"Global" : {
|
"acquireCount" : {
|
"r" : NumberLong(6225014),
|
"w" : NumberLong(4566),
|
"R" : NumberLong(20),
|
"W" : NumberLong(25)
|
},
|
"acquireWaitCount" : {
|
"r" : NumberLong(1)
|
},
|
"timeAcquiringMicros" : {
|
"r" : NumberLong(176)
|
}
|
},
|
"MMAPV1Journal" : {
|
"acquireCount" : {
|
"r" : NumberLong(3125940),
|
"w" : NumberLong(26144567),
|
"R" : NumberLong(72508),
|
"W" : NumberLong(77)
|
},
|
"acquireWaitCount" : {
|
"r" : NumberLong(2058),
|
"w" : NumberLong(2011),
|
"R" : NumberLong(1970),
|
"W" : NumberLong(20)
|
},
|
"timeAcquiringMicros" : {
|
"r" : NumberLong(494344822),
|
"w" : NumberLong(8148242),
|
"R" : NumberLong(186276693),
|
"W" : NumberLong(4214859)
|
}
|
},
|
"Database" : {
|
"acquireCount" : {
|
"r" : NumberLong(3110195),
|
"w" : NumberLong(4501),
|
"R" : NumberLong(27),
|
"W" : NumberLong(65)
|
},
|
"acquireWaitCount" : {
|
"r" : NumberLong(123),
|
"w" : NumberLong(1),
|
"R" : NumberLong(1),
|
"W" : NumberLong(2)
|
},
|
"timeAcquiringMicros" : {
|
"r" : NumberLong(37857023446),
|
"w" : NumberLong(493870),
|
"R" : NumberLong(8302),
|
"W" : NumberLong(259911)
|
}
|
},
|
"Collection" : {
|
"acquireCount" : {
|
"R" : NumberLong(3091273),
|
"W" : NumberLong(4110)
|
},
|
"acquireWaitCount" : {
|
"R" : NumberLong(15317),
|
"W" : NumberLong(164)
|
},
|
"timeAcquiringMicros" : {
|
"R" : NumberLong(7360200399),
|
"W" : NumberLong(31201699)
|
}
|
},
|
"Metadata" : {
|
"acquireCount" : {
|
"R" : NumberLong(4),
|
"W" : NumberLong(367)
|
}
|
},
|
"oplog" : {
|
"acquireCount" : {
|
"w" : NumberLong(391),
|
"R" : NumberLong(21173)
|
},
|
"acquireWaitCount" : {
|
"w" : NumberLong(2),
|
"R" : NumberLong(262)
|
},
|
"timeAcquiringMicros" : {
|
"w" : NumberLong(255158),
|
"R" : NumberLong(10547879)
|
}
|
}
|
},
|
"network" : {
|
"bytesIn" : NumberLong(40962593),
|
"bytesOut" : NumberLong(504146686),
|
"numRequests" : NumberLong(37039)
|
},
|
"opcounters" : {
|
"insert" : 120.0,
|
"query" : 5784.0,
|
"update" : 0.0,
|
"delete" : 0.0,
|
"getmore" : 4130.0,
|
"command" : 27134.0
|
},
|
"opcountersRepl" : {
|
"insert" : 4.0,
|
"query" : 0.0,
|
"update" : 18.0,
|
"delete" : 0.0,
|
"getmore" : 0.0,
|
"command" : 0.0
|
},
|
"repl" : {
|
"setName" : "docuReplicaSet",
|
"setVersion" : 7.0,
|
"ismaster" : true,
|
"secondary" : false,
|
"hosts" : [
|
"mongodbsrv01:27017",
|
"mongodbsrv02:27017",
|
"mongodbsrv03:27017"
|
],
|
"primary" : "mongodbsrv01:27017",
|
"me" : "mongodbsrv01:27017",
|
"electionId" : ObjectId("5a65d4eac297c29a38ecb5b2"),
|
"rbid" : 452501248.0
|
},
|
"storageEngine" : {
|
"name" : "mmapv1"
|
},
|
"writeBacksQueued" : false,
|
"mem" : {
|
"bits" : 64.0,
|
"resident" : 3096.0,
|
"virtual" : 28455.0,
|
"supported" : true,
|
"mapped" : 12395.0,
|
"mappedWithJournal" : 24790.0
|
},
|
"metrics" : {
|
"commands" : {
|
"<UNKNOWN>" : NumberLong(3),
|
"_isSelf" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(2)
|
},
|
"aggregate" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(640)
|
},
|
"buildInfo" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(298)
|
},
|
"collStats" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(3)
|
},
|
"connectionStatus" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(15)
|
},
|
"count" : {
|
"failed" : NumberLong(91),
|
"total" : NumberLong(10032)
|
},
|
"dbStats" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(4)
|
},
|
"findAndModify" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(271)
|
},
|
"getLastError" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(292)
|
},
|
"insert" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(120)
|
},
|
"isMaster" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(3803)
|
},
|
"listCollections" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(19)
|
},
|
"listDatabases" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(1)
|
},
|
"planCacheClear" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(2)
|
},
|
"planCacheListQueryShapes" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(1)
|
},
|
"reIndex" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(1)
|
},
|
"replSetElect" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(1)
|
},
|
"replSetFresh" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(4)
|
},
|
"replSetGetStatus" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(133)
|
},
|
"replSetHeartbeat" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(8226)
|
},
|
"replSetUpdatePosition" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(790)
|
},
|
"saslContinue" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(1716)
|
},
|
"saslStart" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(858)
|
},
|
"serverStatus" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(20)
|
},
|
"usersInfo" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(1)
|
},
|
"whatsmyuri" : {
|
"failed" : NumberLong(0),
|
"total" : NumberLong(3)
|
}
|
},
|
"cursor" : {
|
"timedOut" : NumberLong(1),
|
"open" : {
|
"noTimeout" : NumberLong(74),
|
"pinned" : NumberLong(74),
|
"total" : NumberLong(76)
|
}
|
},
|
"document" : {
|
"deleted" : NumberLong(0),
|
"inserted" : NumberLong(120),
|
"returned" : NumberLong(1221273),
|
"updated" : NumberLong(271)
|
},
|
"getLastError" : {
|
"wtime" : {
|
"num" : 0.0,
|
"totalMillis" : 0.0
|
},
|
"wtimeouts" : NumberLong(0)
|
},
|
"operation" : {
|
"fastmod" : NumberLong(234),
|
"idhack" : NumberLong(0),
|
"scanAndOrder" : NumberLong(10),
|
"writeConflicts" : NumberLong(0)
|
},
|
"queryExecutor" : {
|
"scanned" : NumberLong(558924),
|
"scannedObjects" : NumberLong(530425)
|
},
|
"record" : {
|
"moves" : NumberLong(13)
|
},
|
"repl" : {
|
"apply" : {
|
"batches" : {
|
"num" : 19.0,
|
"totalMillis" : 974.0
|
},
|
"ops" : NumberLong(22)
|
},
|
"buffer" : {
|
"count" : NumberLong(0),
|
"maxSizeBytes" : 268435456.0,
|
"sizeBytes" : NumberLong(0)
|
},
|
"network" : {
|
"bytes" : NumberLong(54827),
|
"getmores" : {
|
"num" : 67.0,
|
"totalMillis" : 280404.0
|
},
|
"ops" : NumberLong(22),
|
"readersCreated" : NumberLong(1)
|
},
|
"preload" : {
|
"docs" : {
|
"num" : 18.0,
|
"totalMillis" : 0.0
|
},
|
"indexes" : {
|
"num" : 511.0,
|
"totalMillis" : 350.0
|
}
|
}
|
},
|
"storage" : {
|
"freelist" : {
|
"search" : {
|
"bucketExhausted" : NumberLong(0),
|
"requests" : NumberLong(163359),
|
"scanned" : NumberLong(0)
|
}
|
}
|
},
|
"ttl" : {
|
"deletedDocuments" : NumberLong(0),
|
"passes" : NumberLong(124)
|
}
|
},
|
"ok" : 1.0
|
}
|
Can anybody help me?