Server refused to start. Had to clean out and do an clean sync.
2016-01-29T18:36:41.852+0000 I CONTROL ***** SERVER RESTARTED ***** 2016-01-29T18:36:41.996+0000 W - [initandlisten] Detected unclean shutdown - /mongodb_data/mongod.lock is not empty. 2016-01-29T18:36:42.046+0000 I JOURNAL [initandlisten] journal dir=/mongodb_data/journal 2016-01-29T18:36:42.046+0000 I JOURNAL [initandlisten] recover begin 2016-01-29T18:36:42.046+0000 I JOURNAL [initandlisten] info no lsn file in journal/ directory 2016-01-29T18:36:42.046+0000 I JOURNAL [initandlisten] recover lsn: 0 2016-01-29T18:36:42.046+0000 I JOURNAL [initandlisten] recover /mongodb_data/journal/j._0 2016-01-29T18:36:42.047+0000 I JOURNAL [initandlisten] recover cleaning up 2016-01-29T18:36:42.047+0000 I JOURNAL [initandlisten] removeJournalFiles 2016-01-29T18:36:42.095+0000 I JOURNAL [initandlisten] recover done 2016-01-29T18:36:42.096+0000 I JOURNAL [initandlisten] preallocating a journal file /mongodb_data/journal/prealloc.0 2016-01-29T18:36:43.361+0000 I JOURNAL [durability] Durability thread started 2016-01-29T18:36:43.361+0000 I JOURNAL [journal writer] Journal writer thread started 2016-01-29T18:36:43.364+0000 I CONTROL [initandlisten] MongoDB starting : pid=1578 port=27017 dbpath=/mongodb_data 64-bit host=ip-10-0-2-103 2016-01-29T18:36:43.364+0000 I CONTROL [initandlisten] db version v3.0.3 2016-01-29T18:36:43.364+0000 I CONTROL [initandlisten] git version: b40106b36eecd1b4407eb1ad1af6bc60593c6105 2016-01-29T18:36:43.364+0000 I CONTROL [initandlisten] OpenSSL version: OpenSSL 1.0.1k-fips 8 Jan 2015 2016-01-29T18:36:43.364+0000 I CONTROL [initandlisten] build info: Linux ip-10-182-86-231 2.6.32-220.el6.x86_64 #1 SMP Wed Nov 9 08:03:13 EST 2011 x86_64 BOOST_LIB_VERSION=1_49 2016-01-29T18:36:43.364+0000 I CONTROL [initandlisten] allocator: tcmalloc 2016-01-29T18:36:43.364+0000 I CONTROL [initandlisten] options: { config: "/etc/mongod.conf", net: { port: 27017, ssl: { PEMKeyFile: "/etc/ssl/mongodb.pem", allowInvalidCertificates: true, allowInvalidHostnames: true, mode: "requireSSL" } }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, replication: { replSetName: "RS-orzovpc-0" }, security: { keyFile: "/var/lib/mongo/rskey" }, storage: { dbPath: "/mongodb_data", engine: "mmapv1", journal: { enabled: true }, mmapv1: { smallFiles: true } }, systemLog: { destination: "file", logAppend: true, path: "/var/log/mongodb/mongod.log" } } 2016-01-29T18:36:43.537+0000 I STORAGE [initandlisten] _getOpenFile() invalid file index requested 37 2016-01-29T18:36:43.537+0000 I - [initandlisten] Invariant failure false src/mongo/db/storage/mmap_v1/mmap_v1_extent_manager.cpp 201 2016-01-29T18:36:43.550+0000 I CONTROL [initandlisten] 0xf6a889 0xf08321 0xeec092 0xd4646b 0xd46605 0xd4663e 0xd4c03a 0xd11fa9 0xd3fab2 0xd402df 0xd4362f 0x915d05 0x7f4631 0x7f93c4 0x7fa96afd4af5 0x7f1bbd ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B6A889"},{"b":"400000","o":"B08321"},{"b":"400000","o":"AEC092"},{"b":"400000","o":"94646B"},{"b":"400000","o":"946605"},{"b":"400000","o":"94663E"},{"b":"400000","o":"94C03A"},{"b":"400000","o":"911FA9"},{"b":"400000","o":"93FAB2"},{"b":"400000","o":"9402DF"},{"b":"400000","o":"94362F"},{"b":"400000","o":"515D05"},{"b":"400000","o":"3F4631"},{"b":"400000","o":"3F93C4"},{"b":"7FA96AFB3000","o":"21AF5"},{"b":"400000","o":"3F1BBD"}],"processInfo":{ "mongodbVersion" : "3.0.3", "gitVersion" : "b40106b36eecd1b4407eb1ad1af6bc60593c6105", "uname" : { "sysname" : "Linux", "release" : "3.10.48-55.140.amzn1.x86_64", "version" : "#1 SMP Wed Jul 9 23:32:19 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "108A63CA14A4BD5E599BAC10885DBD3A85DA5439" }, { "b" : "7FFFA7BD4000", "elfType" : 3, "buildId" : "9C6F128297F4D2216E6CD02B515D59A00C4063AF" }, { "b" : "7FA96C5F3000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "D48D3E6672A77B603B402F661BABF75E90AD570B" }, { "b" : "7FA96C386000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FA96BFA1000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FA96BD99000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "E81013CBFA409053D58A65A0653271AB665A4619" }, { "b" : "7FA96BB95000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "62A8842157C62F95C3069CBF779AFCC26577A99A" }, { "b" : "7FA96B88C000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "66F1CF311C61879639BD3DC0034DEE0D6D042261" }, { "b" : "7FA96B58A000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "5F97F8F8E5024E29717CF35998681F84D4A22D45" }, { "b" : "7FA96B374000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F5D5BC4F1C54" }, { "b" : "7FA96AFB3000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "DF6DA145A649EA093507A635AF383F608E7CE3F2" }, { "b" : "7FA96C80F000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F90843B9087FE91955FEB0355EB0858EF9E97B2" }, { "b" : "7FA96AD70000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9DF61878D8918F25CC74AD01F417FDB051DFE3DA" }, { "b" : "7FA96AA8B000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6F1DB0F811D1B210520443442D4437BC43BF9A80" }, { "b" : "7FA96A888000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "E52249AE6C9865B5C3B9697A57FC92200DA51CF3" }, { "b" : "7FA96A65D000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "F7DF34078FD7BFD684FE46D5F677EEDA1D9B9DC9" }, { "b" : "7FA96A447000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "87B4EBF2183C8EA4AB657212203EFFE6340E2F4F" }, { "b" : "7FA96A23C000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "381960ACAB9C39461D58BDE7B272C4F61BB3582F" }, { "b" : "7FA96A039000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "BF48CD5658DE95CE058C4B828E81C97E2AE19643" }, { "b" : "7FA969E1F000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "6A7DA1CED90F65F27CB7B5BACDBB1C386C05F592" }, { "b" : "7FA969BFE000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "803D7EF21A989677D056E52BAEB9AB5B154FB9D9" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf6a889] mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf08321] mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xB2) [0xeec092] mongod(_ZNK5mongo19MmapV1ExtentManager12_getOpenFileEi+0x8B) [0xd4646b] mongod(_ZNK5mongo19MmapV1ExtentManager12_recordForV1ERKNS_7DiskLocE+0x15) [0xd46605] mongod(_ZNK5mongo19MmapV1ExtentManager11recordForV1ERKNS_7DiskLocE+0xE) [0xd4663e] mongod(_ZNK5mongo17RecordStoreV1Base7dataForEPNS_16OperationContextERKNS_8RecordIdE+0x6A) [0xd4c03a] mongod(_ZNK5mongo38NamespaceDetailsCollectionCatalogEntry13getAllIndexesEPNS_16OperationContextEPSt6vectorISsSaISsEE+0x109) [0xd11fa9] mongod(_ZN5mongo26MMAPV1DatabaseCatalogEntry5_initEPNS_16OperationContextE+0x5A2) [0xd3fab2] mongod(_ZN5mongo26MMAPV1DatabaseCatalogEntryC1EPNS_16OperationContextERKNS_10StringDataES5_bb+0x16F) [0xd402df] mongod(_ZN5mongo12MMAPV1Engine23getDatabaseCatalogEntryEPNS_16OperationContextERKNS_10StringDataE+0x1AF) [0xd4362f] mongod(_ZN5mongo14DatabaseHolder6openDbEPNS_16OperationContextERKNS_10StringDataEPb+0x105) [0x915d05] mongod(_ZN5mongo13initAndListenEi+0xBF1) [0x7f4631] mongod(main+0x134) [0x7f93c4] libc.so.6(__libc_start_main+0xF5) [0x7fa96afd4af5] mongod(+0x3F1BBD) [0x7f1bbd] ----- END BACKTRACE ----- 2016-01-29T18:36:43.550+0000 I - [initandlisten]