2019-02-21T11:28:48.265+0700 I COMMAND [conn51] command mailtarget_tracker.$cmd command: update { update: "link_5c6d1b6cc689d2688b1ab358", ordered: true, writeConcern: { w: "majority" }, $db: "mailtarget_tracker" } numYields:0 reslen:430 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, Database: { acquireCount: { w: 1 } }, Collection: { acquireCount: { w: 1 } } } protocol:op_query 171ms 2019-02-21T11:28:55.004+0700 E - [ftdc] Assertion: Location13538: couldn't open [/proc/22966/stat] Unknown error src/mongo/util/processinfo_li nux.cpp 81 2019-02-21T11:28:56.001+0700 E - [ftdc] Assertion: Location13538: couldn't open [/proc/22966/stat] Unknown error src/mongo/util/processinfo_li nux.cpp 81 2019-02-21T11:28:56.273+0700 E STORAGE [thread798] WiredTiger error (24) [1550723336:270848][22966:0x7f58206a4700], log-server: __directory_list_wor ker, 48: /var/lib/mongo/journal: directory-list: opendir: Too many open files Raw: [1550723336:270848][22966:0x7f58206a4700], log-server: __directory _list_worker, 48: /var/lib/mongo/journal: directory-list: opendir: Too many open files 2019-02-21T11:28:56.273+0700 E STORAGE [thread798] WiredTiger error (24) [1550723336:273379][22966:0x7f58206a4700], log-server: __log_prealloc_once, 484: log pre-alloc server error: Too many open files Raw: [1550723336:273379][22966:0x7f58206a4700], log-server: __log_prealloc_once, 484: log pre-a lloc server error: Too many open files 2019-02-21T11:28:56.273+0700 E STORAGE [thread798] WiredTiger error (24) [1550723336:273405][22966:0x7f58206a4700], log-server: __log_server[60/943] log server error: Too many open files Raw: [1550723336:273405][22966:0x7f58206a4700], log-server: __log_server, 1031: log server error: Too many open files 2019-02-21T11:28:56.273+0700 E STORAGE [thread798] WiredTiger error (-31804) [1550723336:273467][22966:0x7f58206a4700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic Raw: [1550723336:273467][22966:0x7f58206a4700], log-server: __wt_panic, 523: t he process must exit and restart: WT_PANIC: WiredTiger library panic 2019-02-21T11:28:56.273+0700 F - [thread798] Fatal Assertion 50853 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 409 2019-02-21T11:28:56.273+0700 F - [thread798] ***aborting after fassert() failure 2019-02-21T11:28:56.275+0700 F - [WTJournalFlusher] Fatal Assertion 28559 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 65 2019-02-21T11:28:56.275+0700 F - [WTJournalFlusher] ***aborting after fassert() failure 2019-02-21T11:28:56.302+0700 F - [WTJournalFlusher] Got signal: 6 (Aborted). 0x5648f0749281 0x5648f0748499 0x5648f074897d 0x7f5825dda5d0 0x7f5825a34207 0x7f5825a358f8 0x5648eed59ba7 0x5648eee52a17 0x5648eee4b3d1 0x5648eee3157 e 0x5648f019ead1 0x5648f0858f30 0x7f5825dd2dd5 0x7f5825afbead ----- BEGIN BACKTRACE ----- [39/943] {"backtrace":[{"b":"5648EE342000","o":"2407281","s":"_ZN5mongo15printStackTraceERSo"},{"b":"5648EE342000","o":"2406499"},{"b":"5648EE342000","o":"240 697D"},{"b":"7F5825DCB000","o":"F5D0"},{"b":"7F58259FE000","o":"36207","s":"gsignal"},{"b":"7F58259FE000","o":"378F8","s":"abort"},{"b":"5648EE342000 ","o":"A17BA7","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"5648EE342000","o":"B10A17","s":"_ZN5mongo17wtRCToStatus_slowEiPKc"},{"b ":"5648EE342000","o":"B093D1","s":"_ZN5mongo22WiredTigerSessionCache16waitUntilDurableEbb"},{"b":"5648EE342000","o":"AEF57E","s":"_ZN5mongo18WiredTig erKVEngine24WiredTigerJournalFlusher3runEv"},{"b":"5648EE342000","o":"1E5CAD1","s":"_ZN5mongo13BackgroundJob7jobBodyEv"},{"b":"5648EE342000","o":"251 6F30"},{"b":"7F5825DCB000","o":"7DD5"},{"b":"7F58259FE000","o":"FDEAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.6", "gitVersion" : "caa4 2a1f75a56c7643d0b68d3880444375ec42e3", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-862.2.3.el7.x86_64", "version" : "#1 SMP Wed May 9 18:05:47 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "5648EE342000", "elfType" : 3, "buildId" : "E325794241A0BEB620949EC FE27B0CD63A42C8E6" }, { "b" : "7FFD30879000", "elfType" : 3, "buildId" : "C27FD02A7EB073A2EA3FE54791F357FA50D84C2D" }, { "b" : "7F58271F7000", "path" : "/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "9114859D3C4BEC47A03CA321EE367DCA799638CD" }, { "b" : "7F5826FDE000", "path" : "/lib64/libresolv .so.2", "elfType" : 3, "buildId" : "C444AE61E7CBB716FD9C18A0B46A7FE8F4FCF3E5" }, { "b" : "7F5826B7D000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "8BD89856B64DD5189BF075EF574EDF203F93D44A" }, { "b" : "7F582690B000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "A EF5E6F2240B55F90E9DF76CFBB8B9D9F5286583" }, { "b" : "7F5826707000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "357693C8F1F49D93010C4E3 1529C07CDD2BD3D08" }, { "b" : "7F58264FF000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "EFDE2029C9A4A20BE5B8D8AE7E6551FF9B5755D2" }, { "b" : "7F58261FD000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "5B14BE4D749631673523A61074C10959D50F5455" }, { "b" : "7F5825FE7000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "6B4F3D896CD0F06FCB3DEF0245F204ECE3220D7E" }, { "b" : "7F5825DCB000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "96900CB0FF25B26F2BBDF247DE1408242E4773D8" }, { "b" : "7F58259FE000", "path" : "/lib64/libc.so.6"[22/943] pe" : 3, "buildId" : "A644E865A1F672B46A1232F7C275F53B06042696" }, { "b" : "7F5827460000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "bu ildId" : "A527FE72908703C5972AE384E78D1850D1881EE7" }, { "b" : "7F58257CB000", "path" : "/lib64/libidn.so.11", "elfType" : 3, "buildId" : "2B77BBEFFF 65E94F3E0B71A4E89BEB68C4B476C5" }, { "b" : "7F58255A1000", "path" : "/lib64/libssh2.so.1", "elfType" : 3, "buildId" : "39E1832C767A9167A23569784270DD 4B5FD6DC91" }, { "b" : "7F582534F000", "path" : "/lib64/libssl3.so", "elfType" : 3, "buildId" : "2E28F6A705F2ECEA8460D4716D5D1C24B5DDA5E4" }, { "b" : "7F5825128000", "path" : "/lib64/libsmime3.so", "elfType" : 3, "buildId" : "8D0B4010959C321022DF9CE239277A9D7B34A76A" }, { "b" : "7F5824DFB000", "pa th" : "/lib64/libnss3.so", "elfType" : 3, "buildId" : "F5A64BB37FA3972E545EF459A51310F0AB56FA56" }, { "b" : "7F5824BCB000", "path" : "/lib64/libnssut il3.so", "elfType" : 3, "buildId" : "E0705772325A52C3372FFFB8BDE5F786E2E200D6" }, { "b" : "7F58249C7000", "path" : "/lib64/libplds4.so", "elfType" : 3, "buildId" : "084D2194302908913F68B9DCD27DE46FA5B50522" }, { "b" : "7F58247C2000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "799B28 AD9A5460D78376E2C11260F2E858B95DE3" }, { "b" : "7F5824584000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "DE762A28174110911B273E175D5 4F222B313CFE0" }, { "b" : "7F5824337000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "BCC30853830CD911E58700591830DF51ABCBD7BA " }, { "b" : "7F582404E000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "45BAB0BB455BDFA960FDA22E4124CF17B67CC930" }, { "b" : "7F5823 E1B000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "A9B3906192687CC45D483AE3C58C8AF745A6726A" }, { "b" : "7F5823C17000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "B4BE1023D9606A88169DF411BF94AF417D7BA1A0" }, { "b" : "7F5823A08000", "path" : "/lib64/liblber -2.4.so.2", "elfType" : 3, "buildId" : "3192C56CD451E18EB9F29CB045432BA9C738DD29" }, { "b" : "7F58237B3000", "path" : "/lib64/libldap-2.4.so.2", "elf Type" : 3, "buildId" : "F1FADDDE0D21D5F4E2DCADEDD3B85B6E7AAC9883" }, { "b" : "7F582359D000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "B9D5F73428BD6AD68C96986B57BEA3B7CEDB9745" }, { "b" : "7F582338D000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "94B3BCB669126166B77CDCE6092679A6AA2004C8" }, { "b" : "7F5823189000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292 AC58BC30A263" }, { "b" : "7F5822F6C000", "path" : "/lib64/libsasl2.so.3", "elfType" : 3, "buildId" : "E2F2017F821DD1B9D307DA1A9B8014F2941AEB7B" }, { "b" : "7F5822D45000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "D2DD4DA3FDE1477D25BFFF80F3A25FDB541A8179" }, { "b" : "7F5822B0E0 00", "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "740CAD898E29E1F3B73A323CCEC4A7C88911647F" }, { "b" : "7F58228AC000", "path" : "/lib 64/libpcre.so.1", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" }, { "b" : "7F58226A9000", "path" : "/lib64/libfreebl3.so", " elfType" : 3, "buildId" : "B758881F4B6AF6C28C07A1A57713CBD2144628D4" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x5648f0749281] mongod(+0x2406499) [0x5648f0748499] mongod(+0x240697D) [0x5648f074897d] libpthread.so.0(+0xF5D0) [0x7f5825dda5d0] libc.so.6(gsignal+0x37) [0x7f5825a34207] libc.so.6(abort+0x148) [0x7f5825a358f8] mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x5648eed59ba7] mongod(_ZN5mongo17wtRCToStatus_slowEiPKc+0x417) [0x5648eee52a17] mongod(_ZN5mongo22WiredTigerSessionCache16waitUntilDurableEbb+0x701) [0x5648eee4b3d1] mongod(_ZN5mongo18WiredTigerKVEngine24WiredTigerJournalFlusher3runEv+0x11E) [0x5648eee3157e] mongod(_ZN5mongo13BackgroundJob7jobBodyEv+0x131) [0x5648f019ead1] mongod(+0x2516F30) [0x5648f0858f30] libpthread.so.0(+0x7DD5) [0x7f5825dd2dd5] libc.so.6(clone+0x6D) [0x7f5825afbead] ----- END BACKTRACE -----