<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:33:28 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-76739] Archive data missing on evergreen task timeouts</title>
                <link>https://jira.mongodb.org/browse/SERVER-76739</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Possibly related to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-75860&quot; title=&quot;temporarily disable live process hang_analyzer&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-75860&quot;&gt;&lt;del&gt;SERVER-75860&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Example 1:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://parsley.mongodb.com/evergreen/mongodb_mongo_v7.0_enterprise_rhel_80_64_bit_concurrency_sharded_with_stepdowns_and_balancer_0_linux_enterprise_07efe95e998e3f8305c4ec215761e0ec7b9e8012_23_04_24_16_15_19/0/task?bookmarks=0,1710&amp;amp;shareLine=1480&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://parsley.mongodb.com/evergreen/mongodb_mongo_v7.0_enterprise_rhel_80_64_bit_con[&#8230;]12_23_04_24_16_15_19/0/task?bookmarks=0,1710&amp;amp;shareLine=1480&lt;/a&gt;&lt;/p&gt;


&lt;p&gt;&lt;a href=&quot;https://spruce.mongodb.com/task/mongodb_mongo_v7.0_enterprise_rhel_80_64_bit_concurrency_sharded_with_stepdowns_and_balancer_0_linux_enterprise_07efe95e998e3f8305c4ec215761e0ec7b9e8012_23_04_24_16_15_19/logs?execution=0&amp;amp;sortBy=STATUS&amp;amp;sortDir=ASC&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://spruce.mongodb.com/task/mongodb_mongo_v7.0_enterprise_rhel_80_64_bit_concurren[&#8230;]3_04_24_16_15_19/logs?execution=0&amp;amp;sortBy=STATUS&amp;amp;sortDir=ASC&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Example 2:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://parsley.mongodb.com/evergreen/mongodb_mongo_v7.0_enterprise_amazon2_arm64_concurrency_sharded_multi_stmt_txn_with_stepdowns_1_linux_enterprise_07efe95e998e3f8305c4ec215761e0ec7b9e8012_23_04_24_16_15_19/0/task?bookmarks=0,1763&amp;amp;shareLine=1533&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://parsley.mongodb.com/evergreen/mongodb_mongo_v7.0_enterprise_amazon2_arm64_conc[&#8230;]12_23_04_24_16_15_19/0/task?bookmarks=0,1763&amp;amp;shareLine=1533&lt;/a&gt;&lt;/p&gt;</description>
                <environment></environment>
        <key id="2329464">SERVER-76739</key>
            <summary>Archive data missing on evergreen task timeouts</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="tausif.rahman@mongodb.com">Tausif Rahman</assignee>
                                    <reporter username="alex.neben@mongodb.com">Alex Neben</reporter>
                        <labels>
                    </labels>
                <created>Tue, 2 May 2023 00:06:14 +0000</created>
                <updated>Sun, 29 Oct 2023 21:22:10 +0000</updated>
                            <resolved>Tue, 16 May 2023 15:40:13 +0000</resolved>
                                                    <fixVersion>7.1.0-rc0</fixVersion>
                    <fixVersion>6.0.7</fixVersion>
                    <fixVersion>5.0.19</fixVersion>
                    <fixVersion>7.0.0-rc2</fixVersion>
                                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="5452373" author="JIRAUSER1258055" created="Thu, 25 May 2023 15:46:35 +0000"  >&lt;p&gt;Adding some slack messages here b/c some of the above comments were not 100% right &amp;amp; so people can understand this:&lt;br/&gt;
&lt;em&gt;&lt;b&gt;tausif.rahman&lt;/b&gt;&lt;/em&gt;&#160;&lt;/p&gt;

&lt;p&gt;Following up here after stand-up:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;call hang-analyzer on live resmoke.py process
	&lt;ol&gt;
		&lt;li&gt;-&amp;gt; hang-analyzer sends SIGUSR1 to resmoke.py (&lt;a href=&quot;https://github.com/10gen/mongo/blob/6cd0e5c322cf8cc1b24722543a0f7e5604f85ed8/buildscripts/resmokelib/hang_analyzer/process.py#L84&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;)&lt;/li&gt;
	&lt;/ol&gt;
	&lt;/li&gt;
	&lt;li&gt;resmoke.py sigusr1 handler triggered
	&lt;ol&gt;
		&lt;li&gt;-&amp;gt; internally starts another hang-analyzer process on all mongo{d,s} (and other processes) that were spawned by this resmoke.py process. (&lt;a href=&quot;https://github.com/10gen/mongo/blob/5266a96260f70fc4d4d561d505f62bba4c2ff76b/buildscripts/resmokelib/sighandler.py#L155-L162&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;)
		&lt;ol&gt;
			&lt;li&gt;hang-analyer uses the -c and -k flags which indicates to take the core_dump and kill the process&lt;/li&gt;
		&lt;/ol&gt;
		&lt;/li&gt;
	&lt;/ol&gt;
	&lt;/li&gt;
	&lt;li&gt;After the &#8220;resmoke spawned&#8221; hang-analyzer process finishes, resmoke.py execution continues as normal&lt;/li&gt;
&lt;/ol&gt;


&lt;ol&gt;
	&lt;li&gt;
	&lt;ol&gt;
		&lt;li&gt;Upon completion of the currently running test, resmoke does symbolization which can take a lot of time (&lt;a href=&quot;https://github.com/10gen/mongo/blob/5facb9a7c4e98a7636e061544ad9d359302d9847/buildscripts/resmokelib/testing/report.py#L153-L154&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;) &#8212; the script downloads debug symbols&lt;/li&gt;
		&lt;li&gt;We also run some hooks &#8212; but I think those might be fast (&lt;a href=&quot;https://github.com/10gen/mongo/blob/08c2bce1b232839552b3c665362b6bc783835afd/buildscripts/resmokelib/testing/job.py#L202&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;)&lt;/li&gt;
	&lt;/ol&gt;
	&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;If (2) &amp;amp; (3) or more specifically 2.1. &amp;amp; 3.1. take more than 15 minutes, we never hit the archival block (&lt;a href=&quot;https://github.com/10gen/mongo/blob/08c2bce1b232839552b3c665362b6bc783835afd/buildscripts/resmokelib/testing/job.py#L204&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;).&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;We can add a --timeout flag to the hang-analyzer where after the timeout, any pids passed with the -k flag will be killed and the process will finish. In 2.1., we will invoke the hang-analyzer with --timeout 12 so that 3 minutes will remain for resmoke to finish up in (3).&lt;/li&gt;
	&lt;li&gt;We will also not run the symbolizer in 3.1. if SIGUSR1 has been triggered. OR we could also try to archive before symbolization, but I&#8217;m not sure how easy that is with the way things are structured. Either way, the archive should always run successfully with the remaining 3 mins.&lt;br/&gt;
**&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;&lt;em&gt;&lt;b&gt;maxh&lt;/b&gt;&lt;/em&gt;&lt;br/&gt;
Minor clarification for (2.1) the hang analyzer is running in the same resmoke python process. _hang_analyzer.execute() is calling more python code inline and using the subcommand as its entrypoint&lt;/p&gt;

&lt;p&gt;In the first example&#160;&lt;a href=&quot;https://parsley.mongodb.com/evergreen/mongodb_mongo_v7.0_enterprise_rhel_80_64_bit_concurrency_sharded_with_stepdowns_and_balancer_0_linux_enterprise_07efe95e998e3f8305c4ec215761e0ec7b9e8012_23_04_24_16_15_19/0/task?bookmarks=0,1710&amp;amp;shareLine=1480&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://parsley.mongodb.com/evergreen/mongodb_mongo_v7.0_enterprise_rhel_80_64_bit_con[&#8230;]12_23_04_24_16_15_19/0/task?bookmarks=0,1710&amp;amp;shareLine=1480&lt;/a&gt;&#160;the hang analyzer had finished killed the processes and thus had returned from&#160;&lt;tt&gt;_hang_analyzer.execute()&lt;/tt&gt;&#160;already. I suspect the symbolizer service logic could be where this time is getting spent. My thought would be when the sighandler.py code is called we flip a global setting which disables the symbolizer service from running because the 15-minute clock is tickingThere are definitely cases where the hang is induced because a process crashed and so the symbolizer service logic symbolizing the C++ backtrace for ease of searchability and triage is beneficial. However we&apos;ll have a core dump for the crash and so the information is knowable from there and running the symbolizer service logic requires downloading and extracting the debug symbols which like (3a) is saying can be slow&lt;/p&gt;</comment>
                            <comment id="5442988" author="xgen-internal-githook" created="Mon, 22 May 2023 18:26:29 +0000"  >&lt;p&gt;Author: &lt;/p&gt;
{&apos;name&apos;: &apos;Tausif Rahman&apos;, &apos;email&apos;: &apos;tausif.rahman@mongodb.com&apos;, &apos;username&apos;: &apos;trahman1318&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-76739&quot; title=&quot;Archive data missing on evergreen task timeouts&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-76739&quot;&gt;&lt;del&gt;SERVER-76739&lt;/del&gt;&lt;/a&gt; Archive data missing on evergreen task timeouts&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 0249b20d05c58ceb47375413e9eb47e028859489)&lt;br/&gt;
Branch: v6.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/98b7678e4fb28d68834c73a28aa107c370292839&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/98b7678e4fb28d68834c73a28aa107c370292839&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="5442956" author="xgen-internal-githook" created="Mon, 22 May 2023 18:18:43 +0000"  >&lt;p&gt;Author: &lt;/p&gt;
{&apos;name&apos;: &apos;Tausif Rahman&apos;, &apos;email&apos;: &apos;tausif.rahman@mongodb.com&apos;, &apos;username&apos;: &apos;trahman1318&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-76739&quot; title=&quot;Archive data missing on evergreen task timeouts&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-76739&quot;&gt;&lt;del&gt;SERVER-76739&lt;/del&gt;&lt;/a&gt; Archive data missing on evergreen task timeouts&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 0249b20d05c58ceb47375413e9eb47e028859489)&lt;br/&gt;
Branch: v5.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/0e8ad101a75798c5237bb284985cbd6523eb2b6c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/0e8ad101a75798c5237bb284985cbd6523eb2b6c&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="5436156" author="xgen-internal-githook" created="Thu, 18 May 2023 20:28:59 +0000"  >&lt;p&gt;Author: &lt;/p&gt;
{&apos;name&apos;: &apos;Tausif Rahman&apos;, &apos;email&apos;: &apos;tausif.rahman@mongodb.com&apos;, &apos;username&apos;: &apos;trahman1318&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-76739&quot; title=&quot;Archive data missing on evergreen task timeouts&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-76739&quot;&gt;&lt;del&gt;SERVER-76739&lt;/del&gt;&lt;/a&gt; Archive data missing on evergreen task timeouts&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 0249b20d05c58ceb47375413e9eb47e028859489)&lt;br/&gt;
Branch: v7.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/457a802d9e1d03d8ac9e305a30ff210942b7edd9&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/457a802d9e1d03d8ac9e305a30ff210942b7edd9&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="5428646" author="xgen-internal-githook" created="Tue, 16 May 2023 15:39:12 +0000"  >&lt;p&gt;Author: &lt;/p&gt;
{&apos;name&apos;: &apos;Tausif Rahman&apos;, &apos;email&apos;: &apos;tausif.rahman@mongodb.com&apos;, &apos;username&apos;: &apos;trahman1318&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-76739&quot; title=&quot;Archive data missing on evergreen task timeouts&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-76739&quot;&gt;&lt;del&gt;SERVER-76739&lt;/del&gt;&lt;/a&gt; Archive data missing on evergreen task timeouts&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/0249b20d05c58ceb47375413e9eb47e028859489&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/0249b20d05c58ceb47375413e9eb47e028859489&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="5395052" author="JIRAUSER1253549" created="Tue, 2 May 2023 18:04:45 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tausif.rahman%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;tausif.rahman@mongodb.com&quot;&gt;tausif.rahman@mongodb.com&lt;/a&gt; sounds good thanks for clarifying.&lt;/p&gt;</comment>
                            <comment id="5395026" author="JIRAUSER1268551" created="Tue, 2 May 2023 17:56:06 +0000"  >&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Have an archive map which adds an &quot;ArchiveItem&quot; on test creation. Upon test completion, it processes it&apos;s corresponding &quot;ArchiveItem&quot;. In the SIGUSR1 handler, we archive any items that are still in the archive map. &amp;lt;- (I believe this will make handling&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-55756&quot; title=&quot;Allow archiving data for after-suite hooks&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-55756&quot;&gt;SERVER-55756&lt;/a&gt;&#160;easier too)&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;I read this as this was our plan and we don&apos;t need to do an atexit? Was that wrong?&lt;/p&gt;</comment>
                            <comment id="5395025" author="JIRAUSER1258055" created="Tue, 2 May 2023 17:55:44 +0000"  >&lt;p&gt;Sorry &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.moody%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;daniel.moody@mongodb.com&quot;&gt;daniel.moody@mongodb.com&lt;/a&gt; should have been more clear. At first I was considering using the atexit for this, but now I&apos;m scratching the atexit stuff in favor of the more robust SIGUSR1 sighandler which will make &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-55756&quot; title=&quot;Allow archiving data for after-suite hooks&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-55756&quot;&gt;SERVER-55756&lt;/a&gt; easier too.&lt;/p&gt;</comment>
                            <comment id="5395017" author="JIRAUSER1253549" created="Tue, 2 May 2023 17:54:07 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tausif.rahman%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;tausif.rahman@mongodb.com&quot;&gt;tausif.rahman@mongodb.com&lt;/a&gt; Are you saying use atexit for archiving the data files in this ticket, and then there is a follow on ticket to move archiving process into the SIGUSR1 sighandler?&lt;/p&gt;</comment>
                            <comment id="5394920" author="JIRAUSER1268551" created="Tue, 2 May 2023 17:33:16 +0000"  >&lt;p&gt;This sounds great to me!&lt;/p&gt;</comment>
                            <comment id="5394910" author="JIRAUSER1258055" created="Tue, 2 May 2023 17:29:09 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.moody%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;daniel.moody@mongodb.com&quot;&gt;daniel.moody@mongodb.com&lt;/a&gt; - Some follow up: Looks like only &lt;b&gt;&lt;em&gt;non-python&lt;/em&gt;&lt;/b&gt;&#160;processes&#160;&lt;a href=&quot;https://github.com/10gen/mongo/blob/8829adf0f7031c12e87ab7022452a9b057332bd1/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py#L168-L170&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;resume&lt;/a&gt;. Also, I looked into having the hang-analyzer perform the archive but archive relies on some resmoke test infrastructure (ie: whether the test passed or failed &#8211; as well as other more complex logic) which is not currently available via the hang-analyzer.&lt;br/&gt;
------------------&lt;br/&gt;
When we send SIGUSR1 from the hang analyzer, the &lt;a href=&quot;https://github.com/10gen/mongo/blob/5266a96260f70fc4d4d561d505f62bba4c2ff76b/buildscripts/resmokelib/sighandler.py#L28&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;SIGUSR1 handler logic&lt;/a&gt;&#160;runs on the&#160;&lt;b&gt;&lt;em&gt;resmoke.py&lt;/em&gt;&lt;/b&gt;&#160;process &amp;amp; the process is terminated. I did some experimentation &amp;amp; atexit does run successfully right after the SIGUSR1 handler logic &amp;amp; should complete long before the 15 min timeout.&lt;/p&gt;

&lt;p&gt;Timeline of events:&lt;/p&gt;


&lt;p&gt;&lt;b&gt;&lt;em&gt;resmoke.py&lt;/em&gt;&lt;/b&gt;&#160;process: * Starts test suite&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;&lt;a href=&quot;https://github.com/10gen/mongo/blob/08c2bce1b232839552b3c665362b6bc783835afd/buildscripts/resmokelib/testing/job.py#L165&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Execute test&lt;/a&gt;&lt;/li&gt;
&lt;/ul&gt;


&lt;ul&gt;
	&lt;li&gt;
	&lt;ul&gt;
		&lt;li&gt;&lt;a href=&quot;https://github.com/10gen/mongo/blob/08c2bce1b232839552b3c665362b6bc783835afd/buildscripts/resmokelib/testing/job.py#L201&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Hangs before archival in finally block&lt;/a&gt;&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&#160;&lt;br/&gt;
&#8230;2 Hours later Evergreen Timeout procedure starts&#8230;&lt;br/&gt;
&#160;&lt;br/&gt;
&lt;b&gt;&lt;em&gt;hang-analyzer.py&lt;/em&gt;&lt;/b&gt;&#160;process: * &lt;a href=&quot;https://github.com/10gen/mongo/blob/8829adf0f7031c12e87ab7022452a9b057332bd1/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py#L102&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Send SIGUSR1 to resmoke.py process&lt;/a&gt;&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;&lt;a href=&quot;https://github.com/10gen/mongo/blob/8829adf0f7031c12e87ab7022452a9b057332bd1/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py#L108&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Take Core Dumps &amp;amp; Stack Traces&lt;/a&gt;&lt;/li&gt;
&lt;/ul&gt;


&lt;ul&gt;
	&lt;li&gt;
	&lt;ul&gt;
		&lt;li&gt;Evg Timeout procedure times out after 15 mins&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
&lt;/ul&gt;



&lt;p&gt;Next Steps:&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Have an archive map which adds an &quot;ArchiveItem&quot; on test creation. Upon test completion, it processes it&apos;s corresponding &quot;ArchiveItem&quot;. In the SIGUSR1 handler, we archive any items that are still in the archive map. &amp;lt;- (I believe this will make handling &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-55756&quot; title=&quot;Allow archiving data for after-suite hooks&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-55756&quot;&gt;SERVER-55756&lt;/a&gt; easier too)&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="5394344" author="JIRAUSER1253549" created="Tue, 2 May 2023 15:32:56 +0000"  >&lt;p&gt;Currently the testing itself is the one that saves the test data files, but is that required? Does saving the data files require some test specific knowledge or can hang analyzer perform it?&lt;/p&gt;</comment>
                            <comment id="5394257" author="JIRAUSER1268551" created="Tue, 2 May 2023 15:14:00 +0000"  >&lt;p&gt;We can&apos;t get every single thing we need but from a high level I think the correct order of importance should be data files (especially since I am totally guessing they should be quicker), core dumps, then stack traces.&lt;/p&gt;</comment>
                            <comment id="5394208" author="JIRAUSER1253549" created="Tue, 2 May 2023 15:03:26 +0000"  >&lt;p&gt;The hang analyzer will resume the python process after its done, unfortunately it ran out of time, specifically during Symbolization of stacktrace files. I don&apos;t know if atexit will help because the python processes are suspended from sigusr1, and then when the hang analyzer runs out of time, I am not sure how evergreen shuts down the current running tasks/processes? I assume its a hard kill so at exit would not run?&lt;/p&gt;</comment>
                            <comment id="5394138" author="JIRAUSER1258055" created="Tue, 2 May 2023 14:48:25 +0000"  >&lt;p&gt;I see that we archive data files for failling tests/hooks in &lt;a href=&quot;https://github.com/10gen/mongo/blob/08c2bce1b232839552b3c665362b6bc783835afd/buildscripts/resmokelib/testing/job.py#L201&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this&lt;/a&gt; finally block. When the hang analyzer runs, it sends a SIGUSR1 to the python process &amp;amp; it looks like the archive in the finally block is never getting run. The core dumps still get uploaded because that is performed as part of the hang analyzer process. A quick fix would be to add an atexit method which uploads these on resmoke process exit.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>15.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25578"><![CDATA[v7.0]]></customfieldvalue>
    <customfieldvalue key="23470"><![CDATA[v6.0]]></customfieldvalue>
    <customfieldvalue key="21777"><![CDATA[v5.0]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 2 May 2023 03:52:04 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        36 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            36 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alex.neben@mongodb.com</customfieldvalue>
            <customfieldvalue>daniel.moody@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>tausif.rahman@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i278sv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i1pjhk:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_22250" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Special Downgrade Instructions Required</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="23343"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i26uy7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>