<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:30:25 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-75531] EVG tasks wrongly marked as system unresponsive</title>
                <link>https://jira.mongodb.org/browse/SERVER-75531</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Recently I saw an increased number of tasks marked as &lt;tt&gt;SYSTEM UNRESPONSIVE&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;It seems that in some cases those tasks were executing tests that actually failed but the task is wrongly labeled and the real error is swallowed.&lt;/p&gt;

&lt;p&gt;For instance recently the serverless suite &lt;a href=&quot;https://spruce.mongodb.com/task-history/mongodb-mongo-master/serverless?selectedCommit=52851&amp;amp;visibleColumns=enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;keep failing consistently&lt;/a&gt; with &lt;tt&gt;SYSTEM UNRESPONSIVE&lt;/tt&gt;.&lt;br/&gt;
&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/440521/440521_Screenshot+from+2023-03-31+08-19-50.png&quot; height=&quot;363&quot; width=&quot;237&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;I&apos;ve analyzed &lt;a href=&quot;https://spruce.mongodb.com/task/mongodb_mongo_master_enterprise_rhel_80_64_bit_dynamic_all_feature_flags_required_serverless_f72360f2f7f8176580b3ed6afc8cabb17e95f86d_23_03_30_15_23_33/logs?execution=0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this task&lt;/a&gt; and I noticed that &lt;tt&gt;job3&lt;/tt&gt; &lt;a href=&quot;https://parsley.mongodb.com/evergreen/mongodb_mongo_master_enterprise_rhel_80_64_bit_dynamic_all_feature_flags_required_serverless_f72360f2f7f8176580b3ed6afc8cabb17e95f86d_23_03_30_15_23_33/0/task?bookmarks=0,3017&amp;amp;filters=100job.&amp;amp;highlights=fixture_teardown&amp;amp;shareLine=1507&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;was executing&lt;/a&gt; &lt;tt&gt;change_collection_server_stats.js&lt;/tt&gt; (&lt;a href=&quot;https://logkeeper2.build.10gen.cc/build/c46ef7b981f2e12f2abbba0eca4e93a4/test/17513efb951a64e048a0ded5f5dcf3cc&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;jobs logs&lt;/a&gt;) actually failed with:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[js_test:change_collection_server_stats] assert.soon failed (timeout 600000ms): () =&amp;gt; {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[js_test:change_collection_server_stats]     // All change collection entries are removed but one.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[js_test:change_collection_server_stats]     return changeCollection.count() === 1;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[js_test:change_collection_server_stats] } &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;But this error is not reported in the task logs nor in the evergreen UI.&lt;/p&gt;</description>
                <environment></environment>
        <key id="2303148">SERVER-75531</key>
            <summary>EVG tasks wrongly marked as system unresponsive</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="daniel.moody@mongodb.com">Daniel Moody</assignee>
                                    <reporter username="tommaso.tocci@mongodb.com">Tommaso Tocci</reporter>
                        <labels>
                    </labels>
                <created>Fri, 31 Mar 2023 06:28:15 +0000</created>
                <updated>Fri, 19 May 2023 14:36:56 +0000</updated>
                            <resolved>Wed, 19 Apr 2023 16:53:31 +0000</resolved>
                                                    <fixVersion>7.1.0-rc0</fixVersion>
                                    <component>Testing Infrastructure</component>
                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="5359780" author="JIRAUSER1253549" created="Wed, 19 Apr 2023 16:53:31 +0000"  >&lt;p&gt;Closing this as the underlying issues have been spun off to different tickets/projects. In the general case of this issue we have created EVG-19607 for evergreen to ensure the host does not become unresponsive from bad actor tasks.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;For the specific issues that was causing the system unresponsive in this case (the hang analyzer) we have implemented a workaround in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-75860&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-75860&lt;/a&gt; and created a project to develop a more flexible pipeline for analyzing core dumps in evergreen: &lt;a href=&quot;https://jira.mongodb.org/browse/PM-3334&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/PM-3334&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="5326943" author="JIRAUSER1253549" created="Wed, 5 Apr 2023 15:24:04 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=max.hirschhorn%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;max.hirschhorn@mongodb.com&quot;&gt;max.hirschhorn@mongodb.com&lt;/a&gt; I think your right, my changes also incidentally completely disabled the live process analyzer, so that may be the sole culprit for getting OOM killed. I&apos;ll make sure to check just disabling the live process analyzer and it still passes with the existing core dump command.&lt;/p&gt;</comment>
                            <comment id="5326832" author="max.hirschhorn@10gen.com" created="Wed, 5 Apr 2023 14:51:32 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.moody%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;daniel.moody@mongodb.com&quot;&gt;daniel.moody@mongodb.com&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=alexander.neben%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;alexander.neben@mongodb.com&quot;&gt;alexander.neben@mongodb.com&lt;/a&gt;, what experiment was run to confirm the high memory usage is consumed while taking core dumps? The changes to buildscripts/resmokelib/hang&amp;#95;analyzer/dumper.py which Daniel shared in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-75531?focusedCommentId=5321992&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-5321992&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;this earlier comment&lt;/a&gt; would affect all gdb invocations and not only the one for running the &lt;tt&gt;gcore&lt;/tt&gt; command.&lt;/p&gt;

&lt;p&gt;I had added the &amp;#45;&amp;#45;readnever option to the gdb invocation when only taking core dumps to match the behavior of the &lt;tt&gt;gcore&lt;/tt&gt; script in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-72613&quot; title=&quot;Speed up taking core dumps with the hang analyzer&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-72613&quot;&gt;&lt;del&gt;SERVER-72613&lt;/del&gt;&lt;/a&gt;. I would have expected the amount of additional system memory to capture a core dump to be minimal.&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;The hang analyzer dumper should not attach to multiple processes from the same gdb instance. Each attach should be a new separate gdb process executed in serial (to prevent OOM situations).&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Reusing the same gdb process across multiple mongod processes is intentional to avoid reloading the debug symbols multiple times. I&apos;m not sure I understand the rationale here to change that behavior.&lt;/p&gt;</comment>
                            <comment id="5324854" author="JIRAUSER1253549" created="Tue, 4 Apr 2023 21:23:01 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=alexander.neben%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;alexander.neben@mongodb.com&quot;&gt;alexander.neben@mongodb.com&lt;/a&gt; FYI action 2 from above could be done by us. I believe there are ways to launch python subprocesses to disconnect (for the most part) from the parent processes, like new process group flags or similar. maybe its best to figure how to do this at the point resmoke is launched in the evergreen yaml (some bash script probably?).&lt;/p&gt;</comment>
                            <comment id="5324754" author="JIRAUSER1253549" created="Tue, 4 Apr 2023 20:44:07 +0000"  >&lt;p&gt;using gdb to generate the core dump is probably more expensive in terms of memory, I could do some testing to compare other options. The thing I am not sure if there are things about gdb that generate the coredump in a certain way we want? for example the core dump command we use:&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;[&apos;/opt/mongodbtoolchain/v4/bin/gdb&apos;, &apos;-quiet&apos;, &apos;nx&apos;, &apos;-readnever&apos;, &apos;-ex&apos;, &apos;set interactive-mode off&apos;, &apos;-ex&apos;, &apos;set print thread-events off&apos;, &apos;-ex&apos;, &apos;set python print-stack full&apos;, &apos;-ex&apos;, &quot;py sys.path.extend(&lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;/data/mci/746e7dc74b078f99afa4b6ef6d1beff7/src/buildscripts&amp;#39;, &amp;#39;/data/mci/746e7dc74b078f99afa4b6ef6d1beff7/venv/lib/python3.10/site-packages/git/ext/gitdb&amp;#39;, &amp;#39;/data/mci/746e7dc74b078f99afa4b6ef6d1beff7/src&amp;#39;, &amp;#39;/opt/mongodbtoolchain/revisions/11316f1e7b36f08dcdd2ad0640af18f9287876f4/stow/python3-v4.qUz/lib/python310.zip&amp;#39;, &amp;#39;/opt/mongodbtoolchain/revisions/11316f1e7b36f08dcdd2ad0640af18f9287876f4/stow/python3-v4.qUz/lib/python3.10&amp;#39;, &amp;#39;/opt/mongodbtoolchain/revisions/11316f1e7b36f08dcdd2ad0640af18f9287876f4/stow/python3-v4.qUz/lib/python3.10/lib-dynload&amp;#39;, &amp;#39;/data/mci/746e7dc74b078f99afa4b6ef6d1beff7/venv/lib/python3.10/site-packages&amp;#39;, &amp;#39;/data/mci/746e7dc74b078f99afa4b6ef6d1beff7/src&amp;#39;, &amp;#39;/data/mci/746e7dc74b078f99afa4b6ef6d1beff7/src&amp;#39;, &amp;#39;/data/mci/746e7dc74b078f99afa4b6ef6d1beff7/src&amp;#39;&amp;#93;&lt;/span&gt;)&quot;, &apos;-ex&apos;, &apos;source buildscripts/gdb/mongo.py&apos;, &apos;-ex&apos;, &apos;source buildscripts/gdb/mongo_printers.py&apos;, &apos;-ex&apos;, &apos;source buildscripts/gdb/mongo_lock.py&apos;, &apos;-ex&apos;, &apos;attach 11105&apos;, &apos;-ex&apos;, &apos;handle SIGSTOP ignore noprint&apos;, &apos;-ex&apos;, &apos;set scheduler-locking on&apos;, &apos;-ex&apos;, &apos;gcore dump_mongod.11105.core&apos;, &apos;-ex&apos;, &apos;detach&apos;, &apos;-ex&apos;, &apos;set confirm off&apos;, &apos;-ex&apos;, &apos;quit&apos;]&lt;/p&gt;&lt;/blockquote&gt;
</comment>
                            <comment id="5324421" author="JIRAUSER1268551" created="Tue, 4 Apr 2023 19:07:18 +0000"  >&lt;p&gt;I would love to avoid running gdb dumping in serial. I am wondering if we can use something like &lt;a href=&quot;https://man7.org/linux/man-pages/man1/gcore.1.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://man7.org/linux/man-pages/man1/gcore.1.html&lt;/a&gt; to consume less memory? &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=zakhar.kleyman%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;zakhar.kleyman@mongodb.com&quot;&gt;zakhar.kleyman@mongodb.com&lt;/a&gt; who on evergreen side is responsible for the agent? Could either points 1/2 be reasonable?&lt;/p&gt;</comment>
                            <comment id="5324269" author="JIRAUSER1253549" created="Tue, 4 Apr 2023 18:20:00 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=max.hirschhorn%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;max.hirschhorn@mongodb.com&quot;&gt;max.hirschhorn@mongodb.com&lt;/a&gt; &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=alexander.neben%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;alexander.neben@mongodb.com&quot;&gt;alexander.neben@mongodb.com&lt;/a&gt; So just to recap the actions we can take here:&lt;/p&gt;

&lt;p&gt;1. The oom killer should not kill the evergreen agent, or at least respawn a new agent beforehand in that case so logging and other normal operations can continue.&lt;br/&gt;
2. resmoke (or any) subprocesses should not be connected to the agent so that if the subprocess causes an OOM situation the agent is not killed.&lt;br/&gt;
3. The hang analyzer dumper should not attach to multiple processes from the same gdb instance. Each attach should be a new separate gdb process executed in serial (to prevent OOM situations).&lt;/p&gt;</comment>
                            <comment id="5321992" author="JIRAUSER1253549" created="Tue, 4 Apr 2023 01:59:33 +0000"  >&lt;p&gt;So I did test where I split up each core dump to separate gdb process per mongo process, and it looks like it didn&apos;t oom and system unresponsive with &lt;a href=&quot;https://evergreen.mongodb.com/filediff/642b40b70ae606eb62568816/?file_name=buildscripts%2Fresmokelib%2Fhang_analyzer%2Fdumper.py&amp;amp;patch_number=0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this change&lt;/a&gt;. NOTE: I haven&apos;t validated the core files yet.&lt;/p&gt;

&lt;p&gt;Heres the patch: &lt;a href=&quot;https://spruce.mongodb.com/version/642b40b70ae606eb62568816/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://spruce.mongodb.com/version/642b40b70ae606eb62568816/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;I suspect this might be a little slower because there multiple gdb process to be launched and loaded.&lt;/p&gt;</comment>
                            <comment id="5320527" author="JIRAUSER1253549" created="Mon, 3 Apr 2023 18:16:50 +0000"  >&lt;p&gt;disabling the gdb calls in the hang analyzer &lt;a href=&quot;https://spruce.mongodb.com/version/642b0599d1fe070cb3995232/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;gets rid of the system unresponsive&lt;/a&gt; so it seems for sure it is the culprit and every task I have looked at showed a high mem usage (&amp;gt;80%) before the last process list was logged, so most likely it is an OOM situation.&lt;/p&gt;

&lt;p&gt;I don&apos;t understand why OOM works on other processes, but this GDB process causes the agent to get killed? Do we need to use the special process flags to ensure its not related to the agent process?&lt;/p&gt;

&lt;p&gt;In anycase, we should attempt to fix the dumper in the hang analyzer because I think obtaining to core dumps is pretty critical and just OOM killing GDB is not a great solution. Looking at the current dump command, its a single gdb command which attaches; dumps; detaches for each process. Could doing this in a single gdb command cause the memory of each process to be left over? Is there some extra commands we need clear the detached processes memory?&lt;/p&gt;</comment>
                            <comment id="5319355" author="JIRAUSER1253549" created="Mon, 3 Apr 2023 14:16:53 +0000"  >&lt;p&gt;It seems the test is green again, someone must have fixed the hang in the test. I will continue investigating assuming a hang will still cause system unresponsive and try injecting a hang situation or building one of the commits that was hanging.&lt;/p&gt;</comment>
                            <comment id="5316849" author="max.hirschhorn@10gen.com" created="Fri, 31 Mar 2023 21:46:21 +0000"  >&lt;blockquote&gt;
&lt;p&gt;if it was the oom killer we should see some logs? Unless the oom killer logs after the kill? Should the oom killer be made to log warnings and intents for processes about to be killed some time before?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Since EVG-5419, the &lt;tt&gt;evergreen&lt;/tt&gt; agent checks the dmesg ring buffer to surface OOM killer events. However if the &lt;tt&gt;evergreen&lt;/tt&gt; agent is also dying then this message doesn&apos;t get persisted anywhere because after losing contact the Evergreen app server will have the EC2 host terminated. Perhaps the Evergreen app server could instead contact jasper to start up another &lt;tt&gt;evergreen&lt;/tt&gt; agent or run a command through jasper to check the dmesg ring buffer contents before terminating the EC2 machine so we&apos;d have better diagnostics for this scenario?&lt;/p&gt;

&lt;p&gt;The output from the OOM killer would appear in the agent logs (&lt;a href=&quot;https://parsley.mongodb.com/evergreen/mongodb_mongo_master_enterprise_rhel80_gcc_cxx20_pm_1328_experimental_compile_all_050e6d413c622ba0a5bcb519ad047f1d7b5ba9bf_23_03_24_22_42_06/0/agent?bookmarks=0,137&amp;amp;shareLine=37&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;for example&lt;/a&gt;) however the &lt;tt&gt;evergreen&lt;/tt&gt; agent is only monitoring for OOM killer events at specific points around process cleanup. It isn&apos;t doing it continuously and so there&apos;s a fair chance the dmesg output never makes it to Cedar logging (assuming an OOM killer event is what is actually happening here).&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2023/03/26 08:42:40.899] Found an OOM kill (in 0.155 seconds).&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2023/03/26 08:42:40.899] [14294.896189] Out of memory: Kill process 1528 (objcopy) score 52 or sacrifice child&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2023/03/26 08:42:40.899] [14294.905456] Killed process 1528 (objcopy) total-vm:3440984kB, anon-rss:3423572kB, file-rss:0kB, shmem-rss:0kB&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2023/03/26 08:42:40.899] [14370.249355] Out of memory: Kill process 1532 (objcopy) score 52 or sacrifice child&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2023/03/26 08:42:40.899] [14370.258671] Killed process 1532 (objcopy) total-vm:3428908kB, anon-rss:3411384kB, file-rss:0kB, shmem-rss:0kB&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2023/03/26 08:42:40.899] [14461.005377] Out of memory: Kill process 1554 (objcopy) score 60 or sacrifice child&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2023/03/26 08:42:40.899] [14461.015048] Killed process 1554 (objcopy) total-vm:3903836kB, anon-rss:3879516kB, file-rss:4kB, shmem-rss:0kB&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="5316814" author="JIRAUSER1253549" created="Fri, 31 Mar 2023 21:23:02 +0000"  >&lt;p&gt;if it was the oom killer we should see some logs? Unless the oom killer logs after the kill? Should the oom killer be made to log warnings and intents for processes about to be killed some time before?&lt;/p&gt;</comment>
                            <comment id="5316196" author="JIRAUSER1268551" created="Fri, 31 Mar 2023 17:29:00 +0000"  >&lt;p&gt;Right before the crash we see gdb use a significant amount of memory. &quot;80.4&quot; means that gdb is using 80.4% of the system memory.&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2023&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;/&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;03&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;/&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;31&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;09&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;30&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;29.553&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;] ec2-user &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;24829&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;96.6&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;80.4&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;15069196&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;12863416&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; ?   Rl   &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;09&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;07&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;54&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;00&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;21&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;48&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;12971&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; /opt/mongodbtoolchain/v4/bin/gdb --quiet --nx -ex set interactive-mode off -ex set print thread-events off -ex set python print-stack full -ex py sys.path.extend([&lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/data/mci/80ada60dc0cd6c3e9eb0d9b09dfbd28c/src/buildscripts&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/data/mci/80ada60dc0cd6c3e9eb0d9b09dfbd28c/venv/lib/python3.10/site-packages/git/ext/gitdb&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/data/mci/80ada60dc0cd6c3e9eb0d9b09dfbd28c/src&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/opt/mongodbtoolchain/revisions/11316f1e7b36f08dcdd2ad0640af18f9287876f4/stow/python3-v4.qUz/lib/python310.zip&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/opt/mongodbtoolchain/revisions/11316f1e7b36f08dcdd2ad0640af18f9287876f4/stow/python3-v4.qUz/lib/python3.10&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/opt/mongodbtoolchain/revisions/11316f1e7b36f08dcdd2ad0640af18f9287876f4/stow/python3-v4.qUz/lib/python3.10/lib-dynload&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/data/mci/80ada60dc0cd6c3e9eb0d9b09dfbd28c/venv/lib/python3.10/site-packages&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/data/mci/80ada60dc0cd6c3e9eb0d9b09dfbd28c/src&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/data/mci/80ada60dc0cd6c3e9eb0d9b09dfbd28c/src&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;, &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&apos;/data/mci/80ada60dc0cd6c3e9eb0d9b09dfbd28c/src&apos;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;]) -ex source buildscripts/gdb/mongo.py -ex source buildscripts/gdb/mongo_printers.py -ex source buildscripts/gdb/mongo_lock.py -ex set logging file debugger_mongod_1955.log -ex set logging on -ex attach &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;1955&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; -ex handle SIGSTOP ignore noprint -ex info sharedlibrary -ex info threads -ex set logging off -ex echo \nWriting raw stacks to debugger_mongod_1955_raw_stacks.log.\n -ex set logging redirect on -ex set logging file debugger_mongod_1955_raw_stacks.log -ex set logging on -ex thread apply all bt -ex set logging off -ex set logging redirect off -ex set logging file debugger_mongod_1955.log -ex set logging on -ex mongodb-uniqstack mongodb-bt-&lt;/span&gt;&lt;span style=&quot;color: #006699; font-weight: bold; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;if&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;-active -ex set scheduler-locking on -ex mongodb-dump-locks -ex mongodb-show-locks -ex mongodb-waitsfor-graph debugger_waitsfor_mongod_1955.gv -ex mongodb-javascript-stack -ex mongod-dump-sessions -ex mongodb-dump-mutexes -ex mongodb-dump-recovery-units -ex mongodb-dump-storage-engine-info -ex detach -ex set logging off -ex set confirm off -ex quit &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;If we trace up the ppid list to find the spawning parent we find the parent is&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2023&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;/&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;03&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;/&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;31&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;09&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;24&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;29.228&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;] ec2-user  &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;6486&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;1.5&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;0.4&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;874732&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;79572&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; ?        Sl   &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;08&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;15&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;46&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;00&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;01&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;:&lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;01&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  &lt;/span&gt;&lt;span style=&quot;color: #009900; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;5824&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; /opt/evergreen/evergreen agent --api_server=https:&lt;/span&gt;&lt;span style=&quot;color: #008200; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;//evergreen.mongodb.com --mode=host --host_id=i-0ea0e6f8ab465bce2 --host_secret=e415245aa2b4db6e0ca34528fac0002e --provider=ec2-fleet --log_prefix=/data/mci/agent --working_directory=/data/mci --cleanup &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;So maybe the oom killer is going all the way up the chain here and is killing evergreen agent? Maybe it cannot allocate memory and is aborting as max suggested. Either way both suggestions i/ii are great. With i being a higher priority.&lt;/p&gt;</comment>
                            <comment id="5314922" author="max.hirschhorn@10gen.com" created="Fri, 31 Mar 2023 11:18:11 +0000"  >&lt;p&gt;I&apos;m marking this as P2 because I think it is a problem for System Unresponsive tasks to be unmonitored. System Unresponsive tasks mean the Evergreen app server lost contact with the &lt;tt&gt;evergreen&lt;/tt&gt; agent running on the machine. Historically this was an indication the EC2 instance was taken away. Evergreen improved reporting in such situations by adding a HOST&amp;#95;TERMINATED&amp;#95;EXTERNALLY event to the host&apos;s recent events page. Moreover we are no longer using spot instances where this would really happen.&lt;/p&gt;

&lt;p&gt;I investigated two independent slew of System Unresponsive tasks which were recently reported via Slack and noticed the hang analyzer was running at the time in all of the tasks. I suspect gdb is allocating a large amount of memory and either the OOM killer is choosing the &lt;tt&gt;evergreen&lt;/tt&gt; agent as its victim or the &lt;tt&gt;evergreen&lt;/tt&gt; agent is aborting due to being unable to allocate memory. I would like to see &amp;#40;i) repeated System Unresponsive tasks start to be monitored so we don&apos;t miss ongoing test/code failures in the system and (ii) some investigation performed into why the hang analyzer / gdb is triggering this condition.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10520">
                    <name>Problem/Incident</name>
                                            <outwardlinks description="causes">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="2309992">SERVER-75860</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="2309994">SERVER-75862</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="440521" name="Screenshot from 2023-03-31 08-19-50.png" size="127836" author="tommaso.tocci@mongodb.com" created="Fri, 31 Mar 2023 06:20:21 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 31 Mar 2023 11:18:11 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        42 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<a href='https://jira.mongodb.org/browse/DEVPROD-2300'>DEVPROD-2300</a>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>max.hirschhorn@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            42 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alex.neben@mongodb.com</customfieldvalue>
            <customfieldvalue>daniel.moody@mongodb.com</customfieldvalue>
            <customfieldvalue>max.hirschhorn@mongodb.com</customfieldvalue>
            <customfieldvalue>tommaso.tocci@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i22qdj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i1l2s8:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_22250" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Special Downgrade Instructions Required</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="23343"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i22civ:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>