<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:21:57 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-29843] Make oplog queries with point equality on ts field use getOplogStartHack</title>
                <link>https://jira.mongodb.org/browse/SERVER-29843</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Doing point queries on the ts field is slow in the oplog because the oplog does not have any indexes. WT has a hack that allows it to do binary search on the ts field but is only enabled when the query predicate has top level $gt/$gte.&lt;/p&gt;

&lt;p&gt;This is needed by retryable writes in order to do fast lookup on oplog entries when traversing through the write history within a transaction.&lt;/p&gt;</description>
                <environment></environment>
        <key id="397755">SERVER-29843</key>
            <summary>Make oplog queries with point equality on ts field use getOplogStartHack</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="tess.avitabile@mongodb.com">Tess Avitabile</assignee>
                                    <reporter username="randolph@mongodb.com">Randolph Tan</reporter>
                        <labels>
                    </labels>
                <created>Fri, 23 Jun 2017 21:38:22 +0000</created>
                <updated>Mon, 30 Oct 2023 23:15:43 +0000</updated>
                            <resolved>Tue, 7 Nov 2017 18:53:23 +0000</resolved>
                                    <version>3.5.9</version>
                                    <fixVersion>3.6.0-rc4</fixVersion>
                                    <component>Querying</component>
                                        <votes>0</votes>
                                    <watches>12</watches>
                                                                                                                <comments>
                            <comment id="2234493" author="bartle" created="Sat, 4 May 2019 01:22:16 +0000"  >&lt;p&gt;For context, we&apos;re seeing this on a node that was offline for index builds for a while.  There&apos;s sufficient oplog runway but the node is stuck because an oplog query (containing a $gte and $lte on the same timestamp, so effectively an $eq) is timing out, presumably because the oplog replay hack isn&apos;t being applied.&lt;/p&gt;</comment>
                            <comment id="2234492" author="bartle" created="Sat, 4 May 2019 01:13:57 +0000"  >&lt;p&gt;Would it be possible to backport this to 3.4?&lt;/p&gt;</comment>
                            <comment id="1719603" author="tess.avitabile" created="Tue, 7 Nov 2017 18:53:23 +0000"  >&lt;p&gt;Previously the &lt;tt&gt;OplogReplay&lt;/tt&gt; query option was supported over queries with a &lt;tt&gt;$gt&lt;/tt&gt; or &lt;tt&gt;$gte&lt;/tt&gt; predicate over the &lt;tt&gt;ts&lt;/tt&gt; field. Now the &lt;tt&gt;OplogReplay&lt;/tt&gt; query option is supported over queries with a &lt;tt&gt;$gt&lt;/tt&gt;, &lt;tt&gt;$gte&lt;/tt&gt;, or &lt;tt&gt;$eq&lt;/tt&gt; predicate of the &lt;tt&gt;ts&lt;/tt&gt; field, but the value that &lt;tt&gt;ts&lt;/tt&gt; is compared to is required to be a &lt;tt&gt;Timestamp&lt;/tt&gt;.&lt;/p&gt;</comment>
                            <comment id="1719599" author="xgen-internal-githook" created="Tue, 7 Nov 2017 18:49:03 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Tess Avitabile&apos;, &apos;username&apos;: &apos;tessavitabile&apos;, &apos;email&apos;: &apos;tess.avitabile@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29843&quot; title=&quot;Make oplog queries with point equality on ts field use getOplogStartHack&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29843&quot;&gt;&lt;del&gt;SERVER-29843&lt;/del&gt;&lt;/a&gt; Make oplog queries use eq, gt, and gte predicates on ts for scan bounds&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/e37ad56eaad3c0d4c7c38af4ee06eaa1329426cd&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/e37ad56eaad3c0d4c7c38af4ee06eaa1329426cd&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1719546" author="renctan" created="Tue, 7 Nov 2017 18:09:41 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tess.avitabile&quot; class=&quot;user-hover&quot; rel=&quot;tess.avitabile&quot;&gt;tess.avitabile&lt;/a&gt; We can do that. But if you have extra bandwidth, that will be helpful as well. I submitted a patch yesterday using your last diff to evergreen.&lt;/p&gt;</comment>
                            <comment id="1719380" author="tess.avitabile" created="Tue, 7 Nov 2017 16:24:01 +0000"  >&lt;p&gt;There are two TODOs for this work:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=renctan&quot; class=&quot;user-hover&quot; rel=&quot;renctan&quot;&gt;renctan&lt;/a&gt;, should I add the &lt;tt&gt;OplogReplay&lt;/tt&gt; option to &lt;a href=&quot;https://github.com/mongodb/mongo/blob/3f51dc0165f5af762616185e4c219e89b886ccb4/src/mongo/db/transaction_history_iterator.cpp#L53&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this query&lt;/a&gt; now that &lt;tt&gt;OplogReplay&lt;/tt&gt; is supported for point equality queries on the &lt;tt&gt;ts&lt;/tt&gt; field? Or should I just remove this TODO and someone can file a ticket for this follow-up work?&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jack.mulrow&quot; class=&quot;user-hover&quot; rel=&quot;jack.mulrow&quot;&gt;jack.mulrow&lt;/a&gt;, should I enable &lt;a href=&quot;https://github.com/mongodb/mongo/blob/3f51dc0165f5af762616185e4c219e89b886ccb4/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml#L55&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;these tests&lt;/a&gt; in the retryable_writes_jscore_passthrough? Or should this be scheduled as follow-up work?&lt;/p&gt;</comment>
                            <comment id="1693419" author="renctan" created="Mon, 9 Oct 2017 17:52:21 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.storch&quot; class=&quot;user-hover&quot; rel=&quot;david.storch&quot;&gt;david.storch&lt;/a&gt; Hold on on that last comment. I have talked with Mathias and he suggested querying with {$natural: 1} instead to get the oldest oplog entry.&lt;/p&gt;</comment>
                            <comment id="1693043" author="renctan" created="Mon, 9 Oct 2017 14:55:43 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.storch&quot; class=&quot;user-hover&quot; rel=&quot;david.storch&quot;&gt;david.storch&lt;/a&gt; Is it also possible to lift the restriction for $lt, limit: 1 queries as well? I need to be able differentiate between whether an oplog has been rolled over or rolled back if I cannot find it and I plan to use { $lt: &amp;lt;ts&amp;gt; } to perform this check.&lt;/p&gt;</comment>
                            <comment id="1637850" author="david.storch" created="Tue, 1 Aug 2017 18:47:54 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=renctan&quot; class=&quot;user-hover&quot; rel=&quot;renctan&quot;&gt;renctan&lt;/a&gt;, I did a little bit of digging on this and turned up the original ticket associated with the restriction that &lt;tt&gt;oplogReplay&lt;/tt&gt; queries have a &lt;tt&gt;$gt&lt;/tt&gt; or &lt;tt&gt;$gte&lt;/tt&gt; predicate on the &lt;tt&gt;ts&lt;/tt&gt; field: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-13566&quot; title=&quot;Using the OplogReplay flag with extra predicates can yield incorrect results&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-13566&quot;&gt;&lt;del&gt;SERVER-13566&lt;/del&gt;&lt;/a&gt;. This was necessary to fix a correctness issue associated with the &lt;tt&gt;OplogStart&lt;/tt&gt; stage on the MMAPv1 storage engine. However, I believe the check is overly strict and can be safely converted to allow equality predicates in addition to &lt;tt&gt;$gt&lt;/tt&gt; or &lt;tt&gt;$gte&lt;/tt&gt;. The code which makes the check that will have to be relaxed is here:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/281d32e13cc558766e46272dabd953e797abc74e/src/mongo/db/query/get_executor.cpp#L533-L553&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/281d32e13cc558766e46272dabd953e797abc74e/src/mongo/db/query/get_executor.cpp#L533-L553&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;I think this will &quot;just work&quot; on WiredTiger, but you&apos;ll have to make a few more changes for MMAP. In particular, you should probably extract the timestamp from the &lt;tt&gt;$gte&lt;/tt&gt;, &lt;tt&gt;$gt&lt;/tt&gt;, or &lt;tt&gt;$eq&lt;/tt&gt; predicate, and then pass it directly to the &lt;tt&gt;OplogStart&lt;/tt&gt; stage here:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/281d32e13cc558766e46272dabd953e797abc74e/src/mongo/db/query/get_executor.cpp#L573-L574&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/281d32e13cc558766e46272dabd953e797abc74e/src/mongo/db/query/get_executor.cpp#L573-L574&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;This is because an equality predicate as &lt;tt&gt;tsExpr&lt;/tt&gt; will do the wrong thing currently if passed directly into &lt;tt&gt;OplogStart&lt;/tt&gt;. Hopefully that makes sense.&lt;/p&gt;

&lt;p&gt;I&apos;m guessing that you will want to set a limit of 1 on these queries, since the query execution machinery does not know about the oplog&apos;s special semantics. In particular, it does not know that the &lt;tt&gt;ts&lt;/tt&gt; field is unique and monotonically increasing. Therefore, without a limit, once it finds an oplog entry matching the &lt;tt&gt;ts&lt;/tt&gt; it will continue to scan to the end of the oplog. Setting a limit will prevent this.&lt;/p&gt;

&lt;p&gt;I&apos;m moving this ticket to the sharding team backlog, but keeping it in Needs Triage. Let me know if you need any more help on this, and please send the code review my way.&lt;/p&gt;</comment>
                            <comment id="1622164" author="ian@10gen.com" created="Fri, 14 Jul 2017 14:45:51 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.storch&quot; class=&quot;user-hover&quot; rel=&quot;david.storch&quot;&gt;david.storch&lt;/a&gt; reminder to please talk to &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=renctan&quot; class=&quot;user-hover&quot; rel=&quot;renctan&quot;&gt;renctan&lt;/a&gt; about how best to implement.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                        <issuelink>
            <issuekey id="456057">SERVER-31863</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10320">
                    <name>Documented</name>
                                                                <inwardlinks description="is documented by">
                                        <issuelink>
            <issuekey id="456751">DOCS-11002</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="449857">SERVER-31687</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="130807">SERVER-13566</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>10.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>3.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10011"><![CDATA[Minor Change]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 30 Jun 2017 14:46:02 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 40 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_14262" key="com.atlassian.jira.plugin.system.customfieldtypes:datepicker">
                        <customfieldname>End date</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 26 Sep 2017 00:00:00 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-257</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 40 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>35.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bartle</customfieldvalue>
            <customfieldvalue>david.storch@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>ian@mongodb.com</customfieldvalue>
            <customfieldvalue>randolph@mongodb.com</customfieldvalue>
            <customfieldvalue>tess.avitabile@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht9uef:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr9dw7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="1891">Sharding 2017-10-02</customfieldvalue>
    <customfieldvalue id="1924">Query 2017-10-23</customfieldvalue>
    <customfieldvalue id="1952">Query 2017-11-13</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_14261" key="com.atlassian.jira.plugin.system.customfieldtypes:datepicker">
                        <customfieldname>Start date</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 23 Jun 2017 00:00:00 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht9ggv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>