<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:55:27 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-40596] Test that change streams and retryable writes fail gracefully when reading oplog entries from &gt;= 2 versions ago</title>
                <link>https://jira.mongodb.org/browse/SERVER-40596</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;See some discussion below. We should add some testing around a scenario where a user quickly upgrades two major versions but some of our machinery such as change streams or Retryable writes still needs to examine older oplog entries from 2 versions ago.&lt;/p&gt;

&lt;h4&gt;&lt;a name=&quot;OriginalDescription.&quot;&gt;&lt;/a&gt;Original Description.&lt;/h4&gt;

&lt;p&gt;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/0d79175a88ee958722c0ffb276606949e695d028/jstests/multiVersion/sharded_txn_downgrade_cluster.js#L76&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;sharded_txn_downgrade_cluster.js &lt;/a&gt; expects retryable writes read oplog entries from higher FCV after downgrade. When I tried to remove the &quot;h&quot; field of oplog format in FCV 4.2, the test failed with the following error.&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;assert: command failed: {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	&quot;n&quot; : 0,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	&quot;writeErrors&quot; : [&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;		{&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;			&quot;index&quot; : 0,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;			&quot;code&quot; : 40414,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;			&quot;codeName&quot; : &quot;Location40414&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;			&quot;errmsg&quot; : &quot;BSON field &apos;OplogEntryBase.h&apos; is missing but a required field&quot;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;		}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	],&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	&quot;ok&quot; : 1,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	&quot;operationTime&quot; : Timestamp(1553956389, 1),&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	&quot;$clusterTime&quot; : {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;		&quot;clusterTime&quot; : Timestamp(1553956423, 1),&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;		&quot;signature&quot; : {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;			&quot;hash&quot; : BinData(0,&quot;AAAAAAAAAAAAAAAAAAAAAAAAAAA=&quot;),&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;			&quot;keyId&quot; : NumberLong(0)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;		}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;_getErrorWithCode@src/mongo/shell/utils.js:25:13&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;doassert@src/mongo/shell/assert.js:18:14&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;_assertCommandWorked@src/mongo/shell/assert.js:584:17&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;assert.commandWorked@src/mongo/shell/assert.js:674:16&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;assertMultiShardRetryableWriteCanBeRetried@jstests/multiVersion/libs/sharded_txn_upgrade_downgrade_cluster_shared.js:93:5&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;@jstests/multiVersion/sharded_txn_downgrade_cluster.js:76:5&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;@jstests/multiVersion/sharded_txn_downgrade_cluster.js:12:2&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;The parsing of oplog entry in TransactionHistoryIterator is done using IDL parser, which failed on the missing &quot;h&quot; field. This implies we cannot introduce any oplog format change that fails the old IDL parsing. It&apos;s less of a problem if we want to remove a field, because we could keep the field as optional for a release and remove it in the following release, assuming users don&apos;t upgrade / downgrade across multiple releases very soon. However, if we add new fields to the oplog entries used by retryable writes, this will be a problem.&lt;/p&gt;

&lt;p&gt;It would be better to reserve the possibility to add new fields to oplog format. There are 3 options.&lt;br/&gt;
 1. Make OplogEntryBase IDL not strict.&lt;br/&gt;
 2. Add the ability to parse and IDL in a non-strict way.&lt;br/&gt;
 3. Disallow retryable writes across FCV boundary.&lt;/p&gt;

&lt;p&gt;Change stream may have the same issue unless it parses the oplog entries manually. CC &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=charlie.swanson&quot; class=&quot;user-hover&quot; rel=&quot;charlie.swanson&quot;&gt;charlie.swanson&lt;/a&gt;.&lt;/p&gt;</description>
                <environment></environment>
        <key id="736617">SERVER-40596</key>
            <summary>Test that change streams and retryable writes fail gracefully when reading oplog entries from &gt;= 2 versions ago</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="10038" iconUrl="https://jira.mongodb.org/images/icons/subtask.gif" description="">Backlog</status>
                    <statusCategory id="2" key="new" colorName="default"/>
                                    <resolution id="-1">Unresolved</resolution>
                                        <assignee username="backlog-query-execution">Backlog - Query Execution</assignee>
                                    <reporter username="siyuan.zhou@mongodb.com">Siyuan Zhou</reporter>
                        <labels>
                    </labels>
                <created>Thu, 11 Apr 2019 19:52:52 +0000</created>
                <updated>Tue, 6 Dec 2022 03:01:58 +0000</updated>
                                                                            <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="2247717" author="siyuan.zhou@10gen.com" created="Wed, 15 May 2019 17:33:17 +0000"  >&lt;p&gt;To make it clear, by &quot;understands oplog entries from the version before it&quot;, I assume you mean before the lower FCV. Prepared transactions doesn&apos;t have to rely on it in 4.4 and I imagine it won&apos;t.&lt;/p&gt;

&lt;p&gt;I agree with Judah, since change stream and&#160;retryable writes are allowed to fail, different from transactions. I was thinking of a test to check if the&#160;change stream reads a format it doesn&apos;t understand (from 2 versions ago, let&apos;s say), the server will return errors correctly rather than fassert.&lt;/p&gt;</comment>
                            <comment id="2247463" author="judah.schvimer" created="Wed, 15 May 2019 15:03:51 +0000"  >&lt;blockquote&gt;
&lt;p&gt;we don&apos;t have to try very hard to make sure we understand either oplog entries from 2 versions ago or oplog entries from the newer version&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I would go farther as to say I&apos;d love if the code never tried to understand oplog entries from 2 versions ago or a newer version, but I would settle for it behaving gracefully if it must. I would like to test that change streams and retryable writes fail gracefully when reading oplog entries from 2 versions ago or newer versions that they do not understand.&lt;/p&gt;</comment>
                            <comment id="2246795" author="charlie.swanson" created="Tue, 14 May 2019 23:13:35 +0000"  >&lt;p&gt;To clarify a bit more, it is a requirement that the current version understands oplog entries from the version before it. This is required for change streams and for prepared transactions and for retryable writes, since we want all of these to work seamlessly across an upgrade and they all read &quot;back in time&quot; to some extent. It&apos;s my personal opinion (and I think Judah agrees) that we don&apos;t have to try very hard to make sure we understand either oplog entries from 2 versions ago or oplog entries from the newer version. It may be worth asking someone from if a quick double or triple-upgade is something we should spend time thinking about, but I don&apos;t think it is.&lt;/p&gt;

&lt;p&gt;So &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt; am I correct that this is not an urgent concern since you did not end up removing the &quot;h&quot; field from the oplog entry parser? If that&apos;s the case, and knowing the above, I don&apos;t see any actionable request from this ticket?&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt; when you say &quot;Testing is definitely required&quot;, what scenario are you imagining? An oplog entry from 2 versions ago being parsed?&lt;/p&gt;</comment>
                            <comment id="2234261" author="judah.schvimer" created="Fri, 3 May 2019 20:16:15 +0000"  >&lt;p&gt;I think the error message now is confusing to an external user, so maybe we want to try to change it if possible, but that doesn&apos;t sound trivial. Testing is definitely required.&lt;/p&gt;</comment>
                            <comment id="2234170" author="siyuan.zhou@10gen.com" created="Fri, 3 May 2019 19:32:17 +0000"  >&lt;blockquote&gt;&lt;p&gt;Reading old oplog entries is unfortunately a requirement for the foreseeable future.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;By requirement, do you mean it has to succeed or it can fail? If it&apos;s allowed to fail, then OplogEntry throwing an uassert exception as it does today is desired. We would just need to test that. That&apos;s my understanding of &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt;&apos;s saying &quot;fail gracefully&quot;.&lt;/p&gt;</comment>
                            <comment id="2233920" author="charlie.swanson" created="Fri, 3 May 2019 17:18:06 +0000"  >&lt;p&gt;Thanks for summarizing &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt;. So after a lot of discussion here, I think we&apos;ve clarified that reading old oplog entries is unfortunately a requirement for the foreseeable future, correct? Given that, what is the purpose of this ticket? Should we just close it as &quot;Won&apos;t fix&quot;? Should we morph it into a ticket to write some testing around it? &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt; do you follow our reasoning/arguments above and agree?&lt;/p&gt;</comment>
                            <comment id="2233775" author="judah.schvimer" created="Fri, 3 May 2019 15:41:43 +0000"  >&lt;p&gt;After talking with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=charlie.swanson&quot; class=&quot;user-hover&quot; rel=&quot;charlie.swanson&quot;&gt;charlie.swanson&lt;/a&gt;, this seems somewhat unavoidable for change streams and retryable writes (or retrying transactions). We&apos;ll have to make sure those fail gracefully, but in 4.2 those should be the only things that read arbitrarily far backwards in the oplog.&lt;/p&gt;

&lt;p&gt;In 4.4, committing a prepared transaction will be able to read behind the replication commit point, so we&apos;ll have to be careful to test and handle that case when we try to change the oplog (See &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40991&quot; title=&quot;4.4 binary in FCV 4.4 should not be able to read oplog entries generated by 4.2 binaries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40991&quot;&gt;&lt;del&gt;SERVER-40991&lt;/del&gt;&lt;/a&gt;).&lt;/p&gt;</comment>
                            <comment id="2230639" author="judah.schvimer" created="Wed, 1 May 2019 15:56:07 +0000"  >&lt;blockquote&gt;
&lt;p&gt;It&apos;s less of a problem if we want to remove a field, because we could keep the field as optional for a release and remove it in the following release, assuming users don&apos;t upgrade / downgrade across multiple releases very soon. &lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;I don&apos;t love the assumption around speed of upgrade/downgrade here and that we crash if users break that assumption. My understanding is that with this behavior we can read oplog entries across multiple FCV changes, which means that we can read back to arbitrarily old formats of oplog entries. This ties our hands in being able to change the oplog format. If we only ever parse oplog entries in the current and (current-1) FCV, then we can safely change the oplog format across 2 releases using the standard FCV gating process.&lt;/p&gt;</comment>
                            <comment id="2230520" author="charlie.swanson" created="Wed, 1 May 2019 14:44:19 +0000"  >&lt;p&gt;Sorry I think I&apos;m missing something. Why are we talking about multiple versions here? The description seems to only indicate 4.0 and 4.2?&lt;/p&gt;</comment>
                            <comment id="2228404" author="judah.schvimer" created="Mon, 29 Apr 2019 21:38:34 +0000"  >&lt;blockquote&gt;
&lt;p&gt;I think we at least need to allow for newer versions to parse and understand old oplog entries (the &apos;upgrade&apos; half of this).&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;I agree, but it should only be necessary across one version. We should be able to make oplog entry format changes across two versions safely. &lt;/p&gt;</comment>
                            <comment id="2228362" author="charlie.swanson" created="Mon, 29 Apr 2019 21:09:48 +0000"  >&lt;p&gt;I don&apos;t fully understand how frequent or racy this behavior is. Is this something that will happen quite often if using the combination of retryable writes and change streams and downgrading? or something that could only happen if there&apos;s a lag in your change stream and so it resumes using a relatively old resume token? If it&apos;s the latter, this is already a concern we&apos;ve dealt with. For example, if you do a multi-document transaction in 4.0 and then downgrade to 3.6, the change stream will not understand it. I think the same applies for prepared transactions from 4.2 to 4.0. I couldn&apos;t find anything in our documentation, but it seems like a good practice would be to somehow ensure your change streams are caught up before doing the downgrade, or expect your change streams to need to be restarted. If this seems common, I think we should probably pursue a fix in some 4.0 version to allow this possibility and take an appropriate action - seems like it&apos;s just allowing for &apos;h&apos; to be absent here?&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;To Judah&apos;s point:&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;I think it would be preferable to make sure we never read oplog entries across an fCV upgrade/downgrade like we&apos;ve been assuming.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;This would certainly be an easier world to think about, but I think we at least need to allow for newer versions to parse and understand old oplog entries (the &apos;upgrade&apos; half of this). I think we&apos;d already have to do that though for the scenario when we&apos;re in the new binary version but the old FCV?&lt;/p&gt;</comment>
                            <comment id="2220645" author="craig.homa" created="Mon, 22 Apr 2019 15:57:07 +0000"  >&lt;p&gt;Hey &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=charlie.swanson&quot; class=&quot;user-hover&quot; rel=&quot;charlie.swanson&quot;&gt;charlie.swanson&lt;/a&gt;, can you please comment on the priority of this?&lt;/p&gt;</comment>
                            <comment id="2213955" author="judah.schvimer" created="Mon, 15 Apr 2019 17:55:04 +0000"  >&lt;blockquote&gt;
&lt;p&gt;It&apos;s less of a problem if we want to remove a field, because we could keep the field as optional for a release and remove it in the following release, assuming users don&apos;t upgrade / downgrade across multiple releases very soon&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I don&apos;t think this is an assumption we want to make, though if the harm is crashing, maybe we&apos;re ok with that behavior. I think it would be preferable to make sure we never read oplog entries across an fCV upgrade/downgrade like we&apos;ve been assuming.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>13.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25125"><![CDATA[Query Execution]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 15 Apr 2019 17:07:22 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 39 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1951</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>alexander.golin@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 39 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>backlog-query-execution</customfieldvalue>
            <customfieldvalue>charlie.swanson@mongodb.com</customfieldvalue>
            <customfieldvalue>craig.homa@mongodb.com</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>siyuan.zhou@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huu707:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr2grz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hutt9j:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>