<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:46:16 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-18006] Shard replicas crash after 3.0.2 upgrade with &quot;Didn&apos;t find RecordId in WiredTigerRecordStore&quot;</title>
                <link>https://jira.mongodb.org/browse/SERVER-18006</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Upgraded a 4-shard replicated 3.0.1 cluster to 3.0.2. 1 hour later, two of the shard replicas crashed with the following stack trace:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-04-10T20:42:23.166+0000 I -        [repl writer worker 14] Assertion: 28556:Didn&apos;t find RecordId in WiredTigerRecordStore&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-04-10T20:42:23.174+0000 I CONTROL  [repl writer worker 14] &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 0xf6b4a9 0xf0bca1 0xef0eff 0xd835bc 0x92e4d0 0xa2ac15 0xa1caa0 0xbed174 0xbed524 0xbedb5d 0xb1a536 0xc577e1 0xcaea6e 0xcb1255 0xf02abb 0xfb8a04 0x7f68e4785df3 0x7f68e32351ad&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;----- BEGIN BACKTRACE -----&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{&quot;backtrace&quot;:[{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;B6B4A9&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;B0BCA1&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;AF0EFF&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;9835BC&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;52E4D0&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;62AC15&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;61CAA0&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;7ED174&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;7ED524&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;7EDB5D&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;71A536&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;8577E1&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;8AEA6E&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;8B1255&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;B02ABB&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;BB8A04&quot;},{&quot;b&quot;:&quot;7F68E477E000&quot;,&quot;o&quot;:&quot;7DF3&quot;},{&quot;b&quot;:&quot;7F68E313F000&quot;,&quot;o&quot;:&quot;F61AD&quot;}],&quot;processInfo&quot;:{ &quot;mongodbVersion&quot; : &quot;3.0.2&quot;, &quot;gitVersion&quot; : &quot;6201872043ecbbc0a4cc169b5482dcf385fc464f&quot;, &quot;uname&quot; : { &quot;sysname&quot; : &quot;Linux&quot;, &quot;release&quot; : &quot;3.4.73-64.112.amzn1.x86_64&quot;, &quot;version&quot; : &quot;#1 SMP Tue Dec 10 01:50:05 UTC 2013&quot;, &quot;machine&quot; : &quot;x86_64&quot; }, &quot;somap&quot; : [ { &quot;elfType&quot; : 2, &quot;b&quot; : &quot;400000&quot;, &quot;buildId&quot; : &quot;810BAA4B0BDE1EBB076BF6A544BF2724A13FAE84&quot; }, { &quot;b&quot; : &quot;7FFFF29DA000&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;6A7E58423C94E707E8B4A45F098E78D557DD5A7D&quot; }, { &quot;b&quot; : &quot;7F68E477E000&quot;, &quot;path&quot; : &quot;/lib64/libpthread.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;D48D3E6672A77B603B402F661BABF75E90AD570B&quot; }, { &quot;b&quot; : &quot;7F68E4511000&quot;, &quot;path&quot; : &quot;/usr/lib64/libssl.so.10&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;A5C6FCE56D21542B9535D5FBCFE74EBA71D24613&quot; }, { &quot;b&quot; : &quot;7F68E412D000&quot;, &quot;path&quot; : &quot;/lib64/libcrypto.so.10&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;75565934551BECB21A027BE0A3AE28A62AEEB7EE&quot; }, { &quot;b&quot; : &quot;7F68E3F25000&quot;, &quot;path&quot; : &quot;/lib64/librt.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;E81013CBFA409053D58A65A0653271AB665A4619&quot; }, { &quot;b&quot; : &quot;7F68E3D21000&quot;, &quot;path&quot; : &quot;/lib64/libdl.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;62A8842157C62F95C3069CBF779AFCC26577A99A&quot; }, { &quot;b&quot; : &quot;7F68E3A18000&quot;, &quot;path&quot; : &quot;/usr/lib64/libstdc++.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;66F1CF311C61879639BD3DC0034DEE0D6D042261&quot; }, { &quot;b&quot; : &quot;7F68E3716000&quot;, &quot;path&quot; : &quot;/lib64/libm.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;5F97F8F8E5024E29717CF35998681F84D4A22D45&quot; }, { &quot;b&quot; : &quot;7F68E3500000&quot;, &quot;path&quot; : &quot;/lib64/libgcc_s.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;C52958E393BDF8E8D090F36DE0F4E620D8736FBF&quot; }, { &quot;b&quot; : &quot;7F68E313F000&quot;, &quot;path&quot; : &quot;/lib64/libc.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;77E48C0DBAE0843560558F1E51A5FC61602ABC5F&quot; }, { &quot;b&quot; : &quot;7F68E499A000&quot;, &quot;path&quot; : &quot;/lib64/ld-linux-x86-64.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;6F90843B9087FE91955FEB0355EB0858EF9E97B2&quot; }, { &quot;b&quot; : &quot;7F68E2EFC000&quot;, &quot;path&quot; : &quot;/lib64/libgssapi_krb5.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;9DF61878D8918F25CC74AD01F417FDB051DFE3DA&quot; }, { &quot;b&quot; : &quot;7F68E2C17000&quot;, &quot;path&quot; : &quot;/lib64/libkrb5.so.3&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;6F1DB0F811D1B210520443442D4437BC43BF9A80&quot; }, { &quot;b&quot; : &quot;7F68E2A14000&quot;, &quot;path&quot; : &quot;/lib64/libcom_err.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;1A6E97644CC9149C2E1871C6AE1DB51975E78A41&quot; }, { &quot;b&quot; : &quot;7F68E27E9000&quot;, &quot;path&quot; : &quot;/lib64/libk5crypto.so.3&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;F7DF34078FD7BFD684FE46D5F677EEDA1D9B9DC9&quot; }, { &quot;b&quot; : &quot;7F68E25D3000&quot;, &quot;path&quot; : &quot;/lib64/libz.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;87B4EBF2183C8EA4AB657212203EFFE6340E2F4F&quot; }, { &quot;b&quot; : &quot;7F68E23C8000&quot;, &quot;path&quot; : &quot;/lib64/libkrb5support.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;381960ACAB9C39461D58BDE7B272C4F61BB3582F&quot; }, { &quot;b&quot; : &quot;7F68E21C5000&quot;, &quot;path&quot; : &quot;/lib64/libkeyutils.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;BF48CD5658DE95CE058C4B828E81C97E2AE19643&quot; }, { &quot;b&quot; : &quot;7F68E1FAB000&quot;, &quot;path&quot; : &quot;/lib64/libresolv.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;17295EFA7A45ADE184C2A7A3294460BDC8B43277&quot; }, { &quot;b&quot; : &quot;7F68E1D8A000&quot;, &quot;path&quot; : &quot;/usr/lib64/libselinux.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;803D7EF21A989677D056E52BAEB9AB5B154FB9D9&quot; } ] }}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf6b4a9]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf0bca1]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xef0eff]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZNK5mongo21WiredTigerRecordStore7dataForEPNS_16OperationContextERKNS_8RecordIdE+0xDC) [0xd835bc]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZNK5mongo10Collection6docForEPNS_16OperationContextERKNS_8RecordIdE+0x20) [0x92e4d0]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo11IDHackStage4workEPm+0x185) [0xa2ac15]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo11DeleteStage4workEPm+0x70) [0xa1caa0]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0xA4) [0xbed174]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x34) [0xbed524]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbedb5d]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo13deleteObjectsEPNS_16OperationContextEPNS_8DatabaseERKNS_10StringDataENS_7BSONObjENS_12PlanExecutor11YieldPolicyEbbbb+0x226) [0xb1a536]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo4repl21applyOperation_inlockEPNS_16OperationContextEPNS_8DatabaseERKNS_7BSONObjEbb+0x1041) [0xc577e1]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo4repl8SyncTail9syncApplyEPNS_16OperationContextERKNS_7BSONObjEb+0x2EE) [0xcaea6e]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo4repl14multiSyncApplyERKSt6vectorINS_7BSONObjESaIS2_EEPNS0_8SyncTailE+0x65) [0xcb1255]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo10threadpool6Worker4loopERKSs+0x2FB) [0xf02abb]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(+0xBB8A04) [0xfb8a04]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; libpthread.so.0(+0x7DF3) [0x7f68e4785df3]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; libc.so.6(clone+0x6D) [0x7f68e32351ad]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;-----  END BACKTRACE  -----&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-04-10T20:42:23.174+0000 E REPL     [repl writer worker 14] writer worker caught exception:  :: caused by :: 28556 Didn&apos;t find RecordId in WiredTigerRecordStore on: { ts: Timestamp 1428698543000|107, h: 3163678979668444329, v: 2, op: &quot;d&quot;, ns: &quot;abc.xyz&quot;, fromMigrate: true, o: { _id: ObjectId(&apos;5509b725e4b0bd119005d2b6&apos;) } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-04-10T20:42:23.174+0000 I -        [repl writer worker 14] Fatal Assertion 16360&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-04-10T20:42:23.174+0000 I -        [repl writer worker 14] &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;***aborting after fassert() failure&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Second replica same trace! But a different collection and about 5 minutes later in time.&lt;/p&gt;

&lt;p&gt;Both of these failures are preceded by a chunk move (in the mongos logs), a few minutes after. Here is the only line in the MongoS that&apos;s pertaining:&lt;br/&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-04-10T20:37:14.516+0000 I SHARDING [Balancer] moving chunk ns: abc.xyz moving ( ns: abc.xyz, shard: rs_prod1_event_shard2:rs_prod1_event_shard2/p1e2r1.prod.evergage.com:27017,p1e2r2.prod.evergage.com:27017, lastmod: 7|1||000000000000000000000000, min: { user.id: &quot;8812601eb0ca0000&quot; }, max: { user.id: &quot;b0adb576c86b0000&quot; }) rs_prod1_event_shard2:rs_prod1_event_shard2/p1e2r1.prod.evergage.com:27017,p1e2r2.prod.evergage.com:27017 -&amp;gt; rs_prod1_event_shard3:rs_prod1_event_shard3/p1e3r1.prod.evergage.com:27017,p1e3r2.prod.evergage.com:27017&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;A little later in the mongos, two minutes before failure is the only other log line related to this same collection:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-04-10T20:40:09.779+0000 I SHARDING [Balancer] ChunkManager: time to load chunks for abc.xyz: 0ms sequenceNumber: 2667 version: 10|1||54ee6b6d4c0a4ab1d04d2986 based on: 9|1||54ee6b6d4c0a4ab1d04d2986&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;So it seems like this was a result of a chunk move. We turned off the load balancer and ran without crashes for the last two days. Of course, it&apos;s not sustainable to leave the load balancer off.&lt;/p&gt;

&lt;p&gt;Replicas would NOT restart after this crash and kept crashing with the same stack trace on the same collection. They would start w/o the replSet argument, but with replSet, it would fail. We ended up removing the collection locally (started w/o replSet) and that allowed the node to restart as a replication member. (We then dropped that collection clusterwide)&lt;/p&gt;

&lt;p&gt;About an hour before the failure, the entire cluster has been upgraded from 3.0.1 to 3.0.2, specifically the mongodb-linux-x86_64-amazon-3.0.2.tgz release. Here&apos;s the uname -a output:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;$ uname -a&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Linux p1e1r2 3.4.73-64.112.amzn1.x86_64 #1 SMP Tue Dec 10 01:50:05 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="195601">SERVER-18006</key>
            <summary>Shard replicas crash after 3.0.2 upgrade with &quot;Didn&apos;t find RecordId in WiredTigerRecordStore&quot;</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="ramon.fernandez@mongodb.com">Ramon Fernandez Marina</assignee>
                                    <reporter username="oleg@evergage.com">Oleg Rekutin</reporter>
                        <labels>
                    </labels>
                <created>Mon, 13 Apr 2015 14:02:36 +0000</created>
                <updated>Tue, 1 Sep 2015 18:39:16 +0000</updated>
                            <resolved>Tue, 19 May 2015 12:58:38 +0000</resolved>
                                    <version>3.0.2</version>
                                                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>11</watches>
                                                                                                                <comments>
                            <comment id="1019727" author="ramon.fernandez" created="Tue, 1 Sep 2015 18:39:16 +0000"  >&lt;p&gt;Thanks for the update &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=oleg%40evergage.com&quot; class=&quot;user-hover&quot; rel=&quot;oleg@evergage.com&quot;&gt;oleg@evergage.com&lt;/a&gt;, that&apos;s great news! And your message should help other users that may run into this issue, so thanks again.&lt;/p&gt;

&lt;p&gt;Cheers,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1019517" author="oleg@evergage.com" created="Tue, 1 Sep 2015 16:06:10 +0000"  >&lt;p&gt;Ramon, thank you for your response! Indeed, we were able to re-sync all the nodes over time and looks like we have been able to get past this issue, as it has not come up since June 17.&lt;/p&gt;</comment>
                            <comment id="1019444" author="ramon.fernandez" created="Tue, 1 Sep 2015 15:29:32 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=oleg%40evergage.com&quot; class=&quot;user-hover&quot; rel=&quot;oleg@evergage.com&quot;&gt;oleg@evergage.com&lt;/a&gt;, as per &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506?focusedCommentId=943624&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-943624&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;Michael&apos;s message&lt;/a&gt; on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt;, once the bug has occurred it&apos;s not possible to detect it or recover from the lost update.&lt;/p&gt;

&lt;p&gt;If you have a replica set you can try to re-sync from a different node; if the source node was not affected the resync process should succeed and give you a baseline to recover the rest of the nodes.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="942945" author="oleg@evergage.com" created="Wed, 17 Jun 2015 14:34:35 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=ramon.fernandez&quot; class=&quot;user-hover&quot; rel=&quot;ramon.fernandez&quot;&gt;ramon.fernandez&lt;/a&gt;, we hit this problem again. Is there a way to proactively find out every database that was affected by (or has leftover corruption from) &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt;?&lt;/p&gt;</comment>
                            <comment id="917803" author="ramon.fernandez" created="Tue, 19 May 2015 12:58:08 +0000"  >&lt;p&gt;Thanks for getting back to us &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=oleg%40evergage.com&quot; class=&quot;user-hover&quot; rel=&quot;oleg@evergage.com&quot;&gt;oleg@evergage.com&lt;/a&gt;. We&apos;ll close the ticket then &amp;#8211; feel free to re-open (or open a new one) if you see this again.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="916659" author="oleg@evergage.com" created="Mon, 18 May 2015 14:27:14 +0000"  >&lt;p&gt;Dan, apologies for missing your question earlier.&lt;/p&gt;

&lt;p&gt;Yes, I did run this cluster with 3.0.0 at one point. SO indeed it&apos;s quite possible that it&apos;s leftovers from &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;We were forced to turn on back load balancing as it wasn&apos;t too happy unbalanced, but I am happy to report that we have not run into this problem again, while running more load and continuing to run load balancing. The cluster is still running 3.0.2.&lt;/p&gt;

&lt;p&gt;Given all the data points, it&apos;s quite possible that this is a &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt; leftover, and I think we can close this as either a dupe or a non reproducible.&lt;/p&gt;</comment>
                            <comment id="915758" author="ramon.fernandez" created="Fri, 15 May 2015 22:05:45 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=oleg%40evergage.com&quot; class=&quot;user-hover&quot; rel=&quot;oleg@evergage.com&quot;&gt;oleg@evergage.com&lt;/a&gt;, we haven&apos;t heard back from you for some time. If this is still an issue for you, can you please provide the information requested by Dan in the previous comment?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="902889" author="dan@10gen.com" created="Fri, 1 May 2015 04:01:09 +0000"  >&lt;p&gt;Hi Oleg, can you confirm whether or not you were running this cluster with 3.0.0 at one point?  It is possible that the collection was impacted by &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt;, but did not manifest until the chunk migration forced the data to be read.&lt;/p&gt;

&lt;p&gt;Thanks&lt;br/&gt;
Dan&lt;/p&gt;</comment>
                            <comment id="895597" author="ramon.fernandez" created="Wed, 22 Apr 2015 21:45:27 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=oleg%40evergage.com&quot; class=&quot;user-hover&quot; rel=&quot;oleg@evergage.com&quot;&gt;oleg@evergage.com&lt;/a&gt;, I think you&apos;re right about storage problems, so I think it&apos;s safe to rule those out.&lt;/p&gt;

&lt;p&gt;As for &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt; my first impression is that is a different problem with the same symptom: you were running 3.0.1 before and you didn&apos;t see the behavior reported in this ticket &amp;#8211; it only appeared after the upgrade to 3.0.2. If for example we forgot about a corner case in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt; I&apos;d say it&apos;s unlikely that it waited until 3.0.2 to show up.&lt;/p&gt;

&lt;p&gt;That being said, the assertion is the same as in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt; so we&apos;ll keep that in mind during our investigation.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="895586" author="oleg@evergage.com" created="Wed, 22 Apr 2015 21:35:26 +0000"  >&lt;p&gt;Hi Ram&#243;n, these replicas run on top of AWS EBS, and I don&apos;t see any errors in system logs related to disk or IO issues.  I don&apos;t see write or read latency abnormalities for the involved GP2 EBS volume for the failure time period. So I don&apos;t think it&apos;s related to on-disk corruption.&lt;/p&gt;

&lt;p&gt;It&apos;s also strange that two of our nodes experienced this shortly after the 3.0.2 upgrade. Another notch against on-disk corruption.&lt;/p&gt;

&lt;p&gt;Could this be related to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17506&quot; title=&quot;Race between inserts and checkpoints can lose records under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17506&quot;&gt;&lt;del&gt;SERVER-17506&lt;/del&gt;&lt;/a&gt;? That&apos;s listed as fixed in 3.0.1, but I don&apos;t know the nature of the fix: if the problem&apos;s &quot;seed&quot; is planted long before the symptom shows up, then is it possible that running 3.0.0 wrote some bad metadata?&lt;/p&gt;</comment>
                            <comment id="895542" author="ramon.fernandez" created="Wed, 22 Apr 2015 20:57:21 +0000"  >&lt;p&gt;Thanks for uploading the logs &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=oleg%40evergage.com&quot; class=&quot;user-hover&quot; rel=&quot;oleg@evergage.com&quot;&gt;oleg@evergage.com&lt;/a&gt;. One think you may be able to do is to check the health of the disks of the affected nodes to rule out on-disk corruption because of flaky hardware.&lt;/p&gt;

&lt;p&gt;There&apos;s a similar error message in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17881&quot; title=&quot;Primary crashing with &amp;#39;Didn&amp;#39;t find RecordId in WiredTigerRecordStore&amp;#39; while tailing capped collection&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17881&quot;&gt;&lt;del&gt;SERVER-17881&lt;/del&gt;&lt;/a&gt; related to tailing a capped collection, and since that&apos;s the mechanism used for replication that ticket may be related to this one.&lt;/p&gt;

&lt;p&gt;I understand your concerns about turning the balancer back on; we can try to reproduce on our end, but until we do there&apos;s unfortunately not enough information for us to proceed. I&apos;m going to link this ticket to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17881&quot; title=&quot;Primary crashing with &amp;#39;Didn&amp;#39;t find RecordId in WiredTigerRecordStore&amp;#39; while tailing capped collection&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17881&quot;&gt;&lt;del&gt;SERVER-17881&lt;/del&gt;&lt;/a&gt; and close it, but if you find any additional information please post it on this ticket and we&apos;ll reopen it for further investigation.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="882424" author="oleg@evergage.com" created="Thu, 16 Apr 2015 00:22:13 +0000"  >&lt;p&gt;Hi Jason, apologies for the delay in response, &lt;/p&gt;

&lt;p&gt;Yes, both members that suffered the issue were in the secondary state.&lt;/p&gt;

&lt;p&gt;I&apos;ve uploaded both the logs from the time that the servers were started as secondaries as 3.0.2 (this was the first start after the 3.0.2 upgrade) to your private server. You can see that both members were in SECONDARY state.&lt;/p&gt;

&lt;p&gt;Unfortunately, we did not capture the data files for these members at the time of the assertion or shortly thereafter. If this happens again, I will try to capture the data files. But our load balancer is off for the time being and I am not eager to reproduce this problem.&lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Oleg&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="879652" author="rassi@10gen.com" created="Mon, 13 Apr 2015 14:59:28 +0000"  >&lt;p&gt;Hi,&lt;/p&gt;

&lt;p&gt;Sorry to hear that you encountered this problem.  We&apos;ll need additional information to further diagnose the issue:&lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Can you confirm that both of the members that suffered this issue were in state SECONDARY at the time?&lt;/li&gt;
	&lt;li&gt;Could you upload the full mongod log for the two members that encountered this problem, as an attachment to this ticket?&lt;/li&gt;
	&lt;li&gt;Do you have a full backup of the data files for either of these members, at the time of the first assertion failure or shortly thereafter?  If so, would you be willing to upload an archive of the data files as an attachment to the ticket?&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;If you are not able to share the files publicly (or any file exceeds ~100MB in size), note also that you may upload the files to a private server of ours that is accessible only to MongoDB staff members (hit &quot;enter&quot; at the password prompt):&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;scp -P 722 &amp;lt;file(s)&amp;gt; SERVER-18006@www.10gen.com:&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;~ Jason Rassi&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="188009">SERVER-17506</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="193796">SERVER-17881</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>13.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 13 Apr 2015 14:59:28 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        8 years, 24 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            8 years, 24 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>rassi</customfieldvalue>
            <customfieldvalue>oleg@evergage.com</customfieldvalue>
            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrl8i7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hs817r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hsg4a7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>