<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:02:52 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-65493] Ensure the commit queue does not get stuck</title>
                <link>https://jira.mongodb.org/browse/SERVER-65493</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The commit queue should not be stuck regardless of the state of the tests or host issues.&lt;/p&gt;

&lt;p&gt;Generate a list of pain points and paths forward for each one as part of this ticket.&lt;/p&gt;

&lt;p&gt;Notes from Slack&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;OOM unittests caused the agent to be killed, which caused the task to restart&lt;/li&gt;
	&lt;li&gt;2 commit queue versions running concurrently, not 3&lt;/li&gt;
	&lt;li&gt;lint running on rhel80-small, not the dedicated CQ variant&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Solution spaces:&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;More timely reverts&lt;/li&gt;
	&lt;li&gt;Monitor runtime&lt;/li&gt;
	&lt;li&gt;Bump min host count for the commit queue variant&lt;/li&gt;
	&lt;li&gt;More validation pre-commit queue&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment></environment>
        <key id="2023460">SERVER-65493</key>
            <summary>Ensure the commit queue does not get stuck</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13203">Gone away</resolution>
                                        <assignee username="robert.guo@mongodb.com">Robert Guo</assignee>
                                    <reporter username="robert.guo@mongodb.com">Robert Guo</reporter>
                        <labels>
                    </labels>
                <created>Tue, 12 Apr 2022 17:49:46 +0000</created>
                <updated>Fri, 27 Oct 2023 20:45:30 +0000</updated>
                            <resolved>Mon, 13 Mar 2023 20:09:52 +0000</resolved>
                                                                    <component>Testing Infrastructure</component>
                                        <votes>1</votes>
                                    <watches>17</watches>
                                                                                                                <comments>
                            <comment id="5268974" author="jeff.zambory" created="Mon, 13 Mar 2023 20:09:52 +0000"  >&lt;p&gt;Closing this as the issue seems to have gone away - please comment or reach out to me if you think this is still relevant!&lt;/p&gt;</comment>
                            <comment id="5160125" author="jeff.zambory" created="Wed, 1 Feb 2023 20:02:12 +0000"  >&lt;p&gt;I just synced with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=annie.black%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;annie.black@mongodb.com&quot;&gt;annie.black@mongodb.com&lt;/a&gt; on this ticket - the Evergreen team has worked on the commit queue recently and has done several improvements on the experience of using it. The issues described in the above ticket likely should be handled by the improvements the Evergreen team has made. Given that, I&apos;m planning on closing this ticket.&#160;&lt;/p&gt;

&lt;p&gt;Does anyone feel otherwise and there&apos;s still value in keeping this ticket open? Are there still deliverables around the commit queue experience that we want to look into?&lt;/p&gt;</comment>
                            <comment id="4831150" author="robert.guo" created="Thu, 15 Sep 2022 17:14:28 +0000"  >&lt;p&gt;Giving this a bump as there was &lt;a href=&quot;https://spruce.mongodb.com/task/mongodb_mongo_master_commit_queue_run_unittests_patch_830439daf3e0ce06af04c8fa84b53f7b9aa3d759_63232fae850e614dbcd27525_22_09_15_14_35_10/tests?execution=2&amp;amp;sortBy=STATUS&amp;amp;sortDir=ASC&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;another occurrence today&lt;/a&gt; with high memory usage causing the Evergreen agent to be killed.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2022/09/15 16:54:26.208] /data/mci/832be73c4e174ba13e85839811cf5e32/toolchain-builder/tmp/build-gdb.sh-bKV/src/gdb-8.3.1/gdb/utils.c:724: internal-error: virtual memory exhausted: can&apos;t allocate 33554523 bytes.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="4478675" author="varun.ravichandran" created="Tue, 12 Apr 2022 22:13:58 +0000"  >&lt;p&gt;To add some context around this:&lt;/p&gt;

&lt;p&gt;2 commits were running the &lt;tt&gt;run_unittests&lt;/tt&gt; task simultaneously, having passed all other tasks thus far. The top commit on the stack had a bug that caused it to fail this task, while the second commit was bug-free. For some reason (presumably the OOM unittests), the buggy commit did not fail but instead kept retrying the task repeatedly. Meanwhile, the second commit ended up hanging indefinitely and eventually timed out on the task, but was not kicked off the commit queue. Since the commit queue was only running 2 versions simultaneously, all remaining commits on the queue remained blocked until both commits were manually removed from the queue.&lt;/p&gt;

&lt;p&gt;I&apos;m curious whether the OOM was responsible for all of this. Did the task restarts prevent the timeout from firing? Why would the failed agent prevent the second commit from being kicked off the queue once it failed?&lt;/p&gt;

&lt;p&gt;We should ensure that even when commits are run in parallel, the status of one commit should not affect the other.&lt;/p&gt;

&lt;p&gt;I&apos;m also curious whether the high load of commits yesterday contributed to this, and, if so, how we can mitigate those types of situations.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 12 Apr 2022 22:13:58 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        47 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            47 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>jeff.zambory@mongodb.com</customfieldvalue>
            <customfieldvalue>robert.guo@mongodb.com</customfieldvalue>
            <customfieldvalue>varun.ravichandran@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i0qxe7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i09w5w:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="5998">DAG 2022-05-16</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i0qjjj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>