[SERVER-32830] MongoDB Server - Slow query performance problem Created: 22/Jan/18  Updated: 22/Feb/18  Resolved: 23/Jan/18

Status: Closed
Project: Core Server
Component/s: None
Affects Version/s: None
Fix Version/s: None

Type: Question Priority: Major - P3
Reporter: Biagio Tozzi Assignee: Kelsey Schubert
Resolution: Done Votes: 0
Labels: None
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Participants:

 Description   

Hello,
I have a problem with my MongoDB cluster: many queries appear to be queued, slowing the system down.
My MongoDB is composed of three server in replicaset.

Server (Primary) TOP:

top - 15:24:11 up 139 days, 22:14,  6 users,  load average: 99,04, 98,94, 98,86
Tasks: 143 total,   5 running, 138 sleeping,   0 stopped,   0 zombie
%Cpu(s):  6,4 us, 29,4 sy,  0,0 ni,  0,0 id, 57,3 wa,  0,0 hi,  6,9 si,  0,0 st
KiB Mem :  4817956 total,    41648 free,  3246844 used,  1529464 buff/cache
KiB Swap:  8388604 total,  8324416 free,    64188 used.  1273813 avail Mem 
 
  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND                             
24862 mongod    20   0 27,789g 3,001g 209216 R  54,8 65,3 132:12.61 mongod  

db.serverStatus():

{ 
    "host" : "mongodbsrv01", 
    "version" : "3.0.8", 
    "process" : "mongod", 
    "pid" : NumberLong(24862), 
    "uptime" : 8267.0, 
    "uptimeMillis" : NumberLong(8266883), 
    "uptimeEstimate" : 6982.0, 
    "localTime" : ISODate("2018-01-22T14:24:27.341+0000"), 
    "asserts" : {
        "regular" : 0.0, 
        "warning" : 0.0, 
        "msg" : 0.0, 
        "user" : 53.0, 
        "rollovers" : 0.0
    }, 
    "backgroundFlushing" : {
        "flushes" : 137.0, 
        "total_ms" : 10382.0, 
        "average_ms" : 75.78102189781022, 
        "last_ms" : 0.0, 
        "last_finished" : ISODate("2018-01-22T14:23:40.786+0000")
    }, 
    "connections" : {
        "current" : 123.0, 
        "available" : 51077.0, 
        "totalCreated" : NumberLong(994)
    }, 
    "cursors" : {
        "note" : "deprecated, use server status metrics", 
        "clientCursors_size" : 76.0, 
        "totalOpen" : 76.0, 
        "pinned" : 74.0, 
        "totalNoTimeout" : 74.0, 
        "timedOut" : 1.0
    }, 
    "dur" : {
        "commits" : 29.0, 
        "journaledMB" : 0.0, 
        "writeToDataFilesMB" : 0.0, 
        "compression" : 0.0, 
        "commitsInWriteLock" : 0.0, 
        "earlyCommits" : 0.0, 
        "timeMs" : {
            "dt" : 3003.0, 
            "prepLogBuffer" : 0.0, 
            "writeToJournal" : 0.0, 
            "writeToDataFiles" : 0.0, 
            "remapPrivateView" : 0.0, 
            "commits" : 1.0, 
            "commitsInWriteLock" : 0.0
        }
    }, 
    "extra_info" : {
        "note" : "fields vary by platform", 
        "heap_usage_bytes" : 443257136.0, 
        "page_faults" : 2302153.0
    }, 
    "globalLock" : {
        "totalTime" : NumberLong(8267149000), 
        "currentQueue" : {
            "total" : 0.0, 
            "readers" : 0.0, 
            "writers" : 0.0
        }, 
        "activeClients" : {
            "total" : 139.0, 
            "readers" : 96.0, 
            "writers" : 0.0
        }
    }, 
    "locks" : {
        "Global" : {
            "acquireCount" : {
                "r" : NumberLong(6225014), 
                "w" : NumberLong(4566), 
                "R" : NumberLong(20), 
                "W" : NumberLong(25)
            }, 
            "acquireWaitCount" : {
                "r" : NumberLong(1)
            }, 
            "timeAcquiringMicros" : {
                "r" : NumberLong(176)
            }
        }, 
        "MMAPV1Journal" : {
            "acquireCount" : {
                "r" : NumberLong(3125940), 
                "w" : NumberLong(26144567), 
                "R" : NumberLong(72508), 
                "W" : NumberLong(77)
            }, 
            "acquireWaitCount" : {
                "r" : NumberLong(2058), 
                "w" : NumberLong(2011), 
                "R" : NumberLong(1970), 
                "W" : NumberLong(20)
            }, 
            "timeAcquiringMicros" : {
                "r" : NumberLong(494344822), 
                "w" : NumberLong(8148242), 
                "R" : NumberLong(186276693), 
                "W" : NumberLong(4214859)
            }
        }, 
        "Database" : {
            "acquireCount" : {
                "r" : NumberLong(3110195), 
                "w" : NumberLong(4501), 
                "R" : NumberLong(27), 
                "W" : NumberLong(65)
            }, 
            "acquireWaitCount" : {
                "r" : NumberLong(123), 
                "w" : NumberLong(1), 
                "R" : NumberLong(1), 
                "W" : NumberLong(2)
            }, 
            "timeAcquiringMicros" : {
                "r" : NumberLong(37857023446), 
                "w" : NumberLong(493870), 
                "R" : NumberLong(8302), 
                "W" : NumberLong(259911)
            }
        }, 
        "Collection" : {
            "acquireCount" : {
                "R" : NumberLong(3091273), 
                "W" : NumberLong(4110)
            }, 
            "acquireWaitCount" : {
                "R" : NumberLong(15317), 
                "W" : NumberLong(164)
            }, 
            "timeAcquiringMicros" : {
                "R" : NumberLong(7360200399), 
                "W" : NumberLong(31201699)
            }
        }, 
        "Metadata" : {
            "acquireCount" : {
                "R" : NumberLong(4), 
                "W" : NumberLong(367)
            }
        }, 
        "oplog" : {
            "acquireCount" : {
                "w" : NumberLong(391), 
                "R" : NumberLong(21173)
            }, 
            "acquireWaitCount" : {
                "w" : NumberLong(2), 
                "R" : NumberLong(262)
            }, 
            "timeAcquiringMicros" : {
                "w" : NumberLong(255158), 
                "R" : NumberLong(10547879)
            }
        }
    }, 
    "network" : {
        "bytesIn" : NumberLong(40962593), 
        "bytesOut" : NumberLong(504146686), 
        "numRequests" : NumberLong(37039)
    }, 
    "opcounters" : {
        "insert" : 120.0, 
        "query" : 5784.0, 
        "update" : 0.0, 
        "delete" : 0.0, 
        "getmore" : 4130.0, 
        "command" : 27134.0
    }, 
    "opcountersRepl" : {
        "insert" : 4.0, 
        "query" : 0.0, 
        "update" : 18.0, 
        "delete" : 0.0, 
        "getmore" : 0.0, 
        "command" : 0.0
    }, 
    "repl" : {
        "setName" : "docuReplicaSet", 
        "setVersion" : 7.0, 
        "ismaster" : true, 
        "secondary" : false, 
        "hosts" : [
            "mongodbsrv01:27017", 
            "mongodbsrv02:27017", 
            "mongodbsrv03:27017"
        ], 
        "primary" : "mongodbsrv01:27017", 
        "me" : "mongodbsrv01:27017", 
        "electionId" : ObjectId("5a65d4eac297c29a38ecb5b2"), 
        "rbid" : 452501248.0
    }, 
    "storageEngine" : {
        "name" : "mmapv1"
    }, 
    "writeBacksQueued" : false, 
    "mem" : {
        "bits" : 64.0, 
        "resident" : 3096.0, 
        "virtual" : 28455.0, 
        "supported" : true, 
        "mapped" : 12395.0, 
        "mappedWithJournal" : 24790.0
    }, 
    "metrics" : {
        "commands" : {
            "<UNKNOWN>" : NumberLong(3), 
            "_isSelf" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(2)
            }, 
            "aggregate" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(640)
            }, 
            "buildInfo" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(298)
            }, 
            "collStats" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(3)
            }, 
            "connectionStatus" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(15)
            }, 
            "count" : {
                "failed" : NumberLong(91), 
                "total" : NumberLong(10032)
            }, 
            "dbStats" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(4)
            }, 
            "findAndModify" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(271)
            }, 
            "getLastError" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(292)
            }, 
            "insert" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(120)
            }, 
            "isMaster" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(3803)
            }, 
            "listCollections" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(19)
            }, 
            "listDatabases" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(1)
            }, 
            "planCacheClear" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(2)
            }, 
            "planCacheListQueryShapes" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(1)
            }, 
            "reIndex" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(1)
            }, 
            "replSetElect" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(1)
            }, 
            "replSetFresh" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(4)
            }, 
            "replSetGetStatus" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(133)
            }, 
            "replSetHeartbeat" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(8226)
            }, 
            "replSetUpdatePosition" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(790)
            }, 
            "saslContinue" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(1716)
            }, 
            "saslStart" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(858)
            }, 
            "serverStatus" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(20)
            }, 
            "usersInfo" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(1)
            }, 
            "whatsmyuri" : {
                "failed" : NumberLong(0), 
                "total" : NumberLong(3)
            }
        }, 
        "cursor" : {
            "timedOut" : NumberLong(1), 
            "open" : {
                "noTimeout" : NumberLong(74), 
                "pinned" : NumberLong(74), 
                "total" : NumberLong(76)
            }
        }, 
        "document" : {
            "deleted" : NumberLong(0), 
            "inserted" : NumberLong(120), 
            "returned" : NumberLong(1221273), 
            "updated" : NumberLong(271)
        }, 
        "getLastError" : {
            "wtime" : {
                "num" : 0.0, 
                "totalMillis" : 0.0
            }, 
            "wtimeouts" : NumberLong(0)
        }, 
        "operation" : {
            "fastmod" : NumberLong(234), 
            "idhack" : NumberLong(0), 
            "scanAndOrder" : NumberLong(10), 
            "writeConflicts" : NumberLong(0)
        }, 
        "queryExecutor" : {
            "scanned" : NumberLong(558924), 
            "scannedObjects" : NumberLong(530425)
        }, 
        "record" : {
            "moves" : NumberLong(13)
        }, 
        "repl" : {
            "apply" : {
                "batches" : {
                    "num" : 19.0, 
                    "totalMillis" : 974.0
                }, 
                "ops" : NumberLong(22)
            }, 
            "buffer" : {
                "count" : NumberLong(0), 
                "maxSizeBytes" : 268435456.0, 
                "sizeBytes" : NumberLong(0)
            }, 
            "network" : {
                "bytes" : NumberLong(54827), 
                "getmores" : {
                    "num" : 67.0, 
                    "totalMillis" : 280404.0
                }, 
                "ops" : NumberLong(22), 
                "readersCreated" : NumberLong(1)
            }, 
            "preload" : {
                "docs" : {
                    "num" : 18.0, 
                    "totalMillis" : 0.0
                }, 
                "indexes" : {
                    "num" : 511.0, 
                    "totalMillis" : 350.0
                }
            }
        }, 
        "storage" : {
            "freelist" : {
                "search" : {
                    "bucketExhausted" : NumberLong(0), 
                    "requests" : NumberLong(163359), 
                    "scanned" : NumberLong(0)
                }
            }
        }, 
        "ttl" : {
            "deletedDocuments" : NumberLong(0), 
            "passes" : NumberLong(124)
        }
    }, 
    "ok" : 1.0
}

Can anybody help me?



 Comments   
Comment by Kelsey Schubert [ 23/Jan/18 ]

Hi biagio.tozzi,

Thanks for your report. Please note that SERVER project is for reporting bugs or feature suggestions for the MongoDB server. For MongoDB-related support discussion please post on the mongodb-user group or Stack Overflow with the mongodb tag. A question like this involving more discussion would be best posted on the mongodb-users group.

Additionally, support for MongoDB 3.0 will expire in February. Therefore, I would strongly recommend you upgrade to a more recent version of MongoDB, which contains many improvements, which may benefit the performance of your cluster.

Kind regards,
Kelsey

Generated at Thu Feb 08 04:31:25 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.