<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:23:45 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-30421] mongod &quot;Cannot allocate memory&quot; and crash</title>
                <link>https://jira.mongodb.org/browse/SERVER-30421</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We just had a mongod crash after seeing this logged. We use sharding and replication.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.670+0000 E STORAGE  [conn38291] WiredTiger (12) [1501475397:670620][1176:0x7fcb5f447700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.670+0000 E STORAGE  [conn31681] WiredTiger (12) [1501475397:670839][1176:0x7fccd90cc700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.670+0000 E STORAGE  [conn34966] WiredTiger (12) [1501475397:670984][1176:0x7fcc21259700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.682+0000 E STORAGE  [conn35450] WiredTiger (12) [1501475397:682359][1176:0x7fcef4660700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.695+0000 E STORAGE  [WTJournalFlusher] WiredTiger (12) [1501475397:695320][1176:0x7fd1676a8700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.715+0000 E STORAGE  [conn20441] WiredTiger (12) [1501475397:715160][1176:0x7fcf63c4f700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.737+0000 E STORAGE  [conn11932] WiredTiger (12) [1501475397:737176][1176:0x7fd12372e700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.787+0000 I -        [conn38291] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.787+0000 I -        [conn11932] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.787+0000 I -        [conn31681] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.788+0000 I -        [conn34966] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.787+0000 I -        [conn20441] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.788+0000 I -        [WTJournalFlusher] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.788+0000 I -        [conn35450] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.821+0000 E STORAGE  [conn31327] WiredTiger (12) [1501475397:821140][1176:0x7fcceda14700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.821+0000 I -        [conn31327] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.827+0000 E STORAGE  [conn33938] WiredTiger (12) [1501475397:827764][1176:0x7fcc58fd3700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.827+0000 I -        [conn33938] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.835+0000 E STORAGE  [conn37020] WiredTiger (12) [1501475397:835110][1176:0x7fcbaa7f6700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.835+0000 I -        [conn37020] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.864+0000 E STORAGE  [conn11921] WiredTiger (12) [1501475397:864452][1176:0x7fd124239700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.864+0000 I -        [conn11921] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.901+0000 E STORAGE  [conn36886] WiredTiger (12) [1501475397:901889][1176:0x7fcbb2b79700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.901+0000 I -        [conn36886] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.926+0000 E STORAGE  [conn37963] WiredTiger (12) [1501475397:926271][1176:0x7fcb72779700], WT_CONNECTION.open_session: only configured to support 20020 sessions (including 20 additional internal sessions): Cannot allocate memory&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.926+0000 I -        [conn37963] Invariant failure: conn-&amp;gt;open_session(conn, NULL, &quot;isolation=snapshot&quot;, &amp;amp;_session) resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 54&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.971+0000 I CONTROL  [WTJournalFlusher] &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 0x1556b32 0x14ee2a4 0x14d4d53 0x126e8fb 0x126f665 0x126fd46 0x125aba3 0x14d7b77 0x1d00200 0x7fd16ca846ba 0x7fd16c7ba3dd&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;----- BEGIN BACKTRACE -----&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{&quot;backtrace&quot;:[{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;1156B32&quot;,&quot;s&quot;:&quot;_ZN5mongo15printStackTraceERSo&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;10EE2A4&quot;,&quot;s&quot;:&quot;_ZN5mongo10logContextEPKc&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;10D4D53&quot;,&quot;s&quot;:&quot;_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;E6E8FB&quot;,&quot;s&quot;:&quot;_ZN5mongo17WiredTigerSessionC1EP15__wt_connectionmm&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;E6F665&quot;,&quot;s&quot;:&quot;_ZN5mongo22WiredTigerSessionCache10getSessionEv&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;E6FD46&quot;,&quot;s&quot;:&quot;_ZN5mongo22WiredTigerSessionCache16waitUntilDurableEb&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;E5ABA3&quot;,&quot;s&quot;:&quot;_ZN5mongo18WiredTigerKVEngine24WiredTigerJournalFlusher3runEv&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;10D7B77&quot;,&quot;s&quot;:&quot;_ZN5mongo13BackgroundJob7jobBodyEv&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;1900200&quot;},{&quot;b&quot;:&quot;7FD16CA7D000&quot;,&quot;o&quot;:&quot;76BA&quot;},{&quot;b&quot;:&quot;7FD16C6B3000&quot;,&quot;o&quot;:&quot;1073DD&quot;,&quot;s&quot;:&quot;clone&quot;}],&quot;processInfo&quot;:{ &quot;mongodbVersion&quot; : &quot;3.2.16&quot;, &quot;gitVersion&quot; : &quot;056bf45128114e44c5358c7a8776fb582363e094&quot;, &quot;compiledModules&quot; : [], &quot;uname&quot; : { &quot;sysname&quot; : &quot;Linux&quot;, &quot;release&quot; : &quot;4.4.0-87-generic&quot;, &quot;version&quot; : &quot;#110-Ubuntu SMP Tue Jul 18 12:55:35 UTC 2017&quot;, &quot;machine&quot; : &quot;x86_64&quot; }, &quot;somap&quot; : [ { &quot;elfType&quot; : 2, &quot;b&quot; : &quot;400000&quot;, &quot;buildId&quot; : &quot;B4C77D1B42936B23E28A2739927CB25274DB2D96&quot; }, { &quot;b&quot; : &quot;7FFC92910000&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;D4057BD712DABB1A8B5F4666C1CA1CD47E70FBA1&quot; }, { &quot;b&quot; : &quot;7FD16DA09000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libssl.so.1.0.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;675F454AD6FD0B6CA2E41127C7B98079DA37F7B6&quot; }, { &quot;b&quot; : &quot;7FD16D5C5000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libcrypto.so.1.0.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;2DA08A7E5BF610030DD33B70DB951399626B7496&quot; }, { &quot;b&quot; : &quot;7FD16D3BD000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/librt.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;F951C1E0765FCAE48F82CAFE35D1ADD36D6C9AF9&quot; }, { &quot;b&quot; : &quot;7FD16D1B9000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libdl.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;0FC788F0861846257B5F1773FBD438E95DFC1032&quot; }, { &quot;b&quot; : &quot;7FD16CEB0000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libm.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;FF7A33D389E756CA381A8189291A968EA5E1F4F8&quot; }, { &quot;b&quot; : &quot;7FD16CC9A000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libgcc_s.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434&quot; }, { &quot;b&quot; : &quot;7FD16CA7D000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libpthread.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;27F189EF8DB8C3734C6A678E6EF3CB0B206D58B2&quot; }, { &quot;b&quot; : &quot;7FD16C6B3000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libc.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;088A6E00A1814622219F346B41E775B8DD46C518&quot; }, { &quot;b&quot; : &quot;7FD16DC72000&quot;, &quot;path&quot; : &quot;/lib64/ld-linux-x86-64.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;9157F205547F0EB588E2AB1F2F120B74253A43EA&quot; } ] }}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1556b32]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo10logContextEPKc+0x174) [0x14ee2a4]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0xF3) [0x14d4d53]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo17WiredTigerSessionC1EP15__wt_connectionmm+0x9B) [0x126e8fb]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo22WiredTigerSessionCache10getSessionEv+0xD5) [0x126f665]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo22WiredTigerSessionCache16waitUntilDurableEb+0x2F6) [0x126fd46]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo18WiredTigerKVEngine24WiredTigerJournalFlusher3runEv+0xB3) [0x125aba3]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo13BackgroundJob7jobBodyEv+0x1B7) [0x14d7b77]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(+0x1900200) [0x1d00200]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; libpthread.so.0(+0x76BA) [0x7fd16ca846ba]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; libc.so.6(clone+0x6D) [0x7fd16c7ba3dd]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;-----  END BACKTRACE  -----&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-07-31T04:29:57.975+0000 I -        [WTJournalFlusher] &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;***aborting after invariant() failure&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment>Ubuntu 16.04, 64 bit.&lt;br/&gt;
&lt;br/&gt;
~60 GB RAM, 8 CPUs.&lt;br/&gt;
&lt;br/&gt;
ii  mongodb-org                        3.2.16                                     amd64        MongoDB open source document-oriented database system (metapackage)&lt;br/&gt;
ii  mongodb-org-mongos                 3.2.16                                     amd64        MongoDB sharded cluster query router&lt;br/&gt;
ii  mongodb-org-server                 3.2.16                                     amd64        MongoDB database server&lt;br/&gt;
ii  mongodb-org-shell                  3.2.16                                     amd64        MongoDB shell client&lt;br/&gt;
ii  mongodb-org-tools                  3.2.16                                     amd64        MongoDB tools&lt;br/&gt;
</environment>
        <key id="410306">SERVER-30421</key>
            <summary>mongod &quot;Cannot allocate memory&quot; and crash</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="dmitry.agranat@mongodb.com">Dmitry Agranat</assignee>
                                    <reporter username="tch@virtall.com">Tomasz Chmielewski</reporter>
                        <labels>
                    </labels>
                <created>Mon, 31 Jul 2017 05:00:47 +0000</created>
                <updated>Wed, 6 Sep 2017 19:43:00 +0000</updated>
                            <resolved>Tue, 8 Aug 2017 13:42:40 +0000</resolved>
                                    <version>3.2.16</version>
                                                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="1642711" author="dmitry.agranat" created="Tue, 8 Aug 2017 07:44:20 +0000"  >&lt;p&gt;Thanks for reporting this behavior &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tch%40virtall.com&quot; class=&quot;user-hover&quot; rel=&quot;tch@virtall.com&quot;&gt;tch@virtall.com&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;We are aware that the system was overloaded at the time of crash. Still, in our opinion, it shouldn&apos;t cause mongod crash (especially that there was no OOM-killer).&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;We are working on improving this in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30462&quot; title=&quot;Coordinate connection and WT session limits&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30462&quot;&gt;&lt;del&gt;SERVER-30462&lt;/del&gt;&lt;/a&gt;. Please feel free to vote and watch &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30462&quot; title=&quot;Coordinate connection and WT session limits&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30462&quot;&gt;&lt;del&gt;SERVER-30462&lt;/del&gt;&lt;/a&gt; for updates.&lt;/p&gt;

&lt;p&gt;Kind Regards,&lt;br/&gt;
Dima&lt;/p&gt;
</comment>
                            <comment id="1641572" author="tch@virtall.com" created="Mon, 7 Aug 2017 02:47:23 +0000"  >&lt;p&gt;Thanks for your analysis.&lt;/p&gt;

&lt;p&gt;To clarify:&lt;/p&gt;

&lt;p&gt;&quot;at 04:33 the server was stopped&quot;.&lt;/p&gt;

&lt;p&gt;It wasn&apos;t stopped by system administrator. It crashed, as initial log shows.&lt;/p&gt;

&lt;p&gt;We are aware that the system was overloaded at the time of crash. Still, in our opinion, it shouldn&apos;t cause mongod crash (especially that there was no OOM-killer).&lt;/p&gt;</comment>
                            <comment id="1641371" author="dmitry.agranat" created="Sun, 6 Aug 2017 12:11:20 +0000"  >
&lt;p&gt;Thank you for the clarification &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tch%40virtall.com&quot; class=&quot;user-hover&quot; rel=&quot;tch@virtall.com&quot;&gt;tch@virtall.com&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;Here is the analysis of the event:&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Summary&lt;/b&gt;:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;Due to a shift in the workload to a different part of the dataset that is not in FS cache, most of the reads are done directly from the disk and disk becomes the bottleneck. This slows down the server as most of the CPU resources are just waiting for I/O.&lt;/li&gt;
	&lt;li&gt;When the server slows down, but the rate of incoming work to the application does not, the application open more connections to queue the requests.&lt;/li&gt;
	&lt;li&gt;At this point, mongod itself is starting to use more memory. This, in turn, is due to the increase in connections, via cursors, stack, etc. This forces the o/s to reduce FS cache, which further exacerbates the FS cache miss issue.&lt;/li&gt;
	&lt;li&gt;For about an hour, between B and the reported crash at 04:29, the WiredTiger &lt;tt&gt;open sessions&lt;/tt&gt; number grows until we reached it&#8217;s maximum defined of 20000. Too many open cursors results in too many open sessions and the system fasserts. Please note that the reported error message is misleading: &lt;tt&gt;Cannot allocate memory&lt;/tt&gt; as we still have available memory. This should be &lt;tt&gt;Cannot allocate resource&lt;/tt&gt; to better reflect this event. I&#8217;ve filed &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3497&quot; title=&quot;Improve logging message when hitting the WT session limits&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3497&quot;&gt;&lt;del&gt;SERVER-30493&lt;/del&gt;&lt;/a&gt; to address this.&lt;/li&gt;
	&lt;li&gt;3 minutes later, at 04:33 the server was stopped and changed to i2.4xlarge Instance which has doubled the CPU and RAM.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&lt;b&gt;Overall event analysis&lt;/b&gt;:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/162606/162606_overall_event.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;There is a significant increase in &lt;tt&gt;wt cache application threads page read from disk to cache time&lt;/tt&gt; metric - indicating that the reads were slower following the incident, either because the disk slowed down, or because the pages requested were slower to be read, or because there are more read requests and/or more bytes being read. This contributes to the overall slow down, and the total time waiting here is quite high (all 128 read threads are waiting).&lt;/li&gt;
	&lt;li&gt;As the event progresses, there is a gradual decrease in the &lt;tt&gt;wt block-manager bytes read&lt;/tt&gt; metric. This metric stands for the FS reads and stays rather stable until the event.&lt;/li&gt;
	&lt;li&gt;There is an increase in the &lt;tt&gt;tcmalloc derived allocated minus wt cache&lt;/tt&gt; metric. This is the amount of memory in the heap that is allocated and in use by the application. The &lt;tt&gt;allocated minus cache&lt;/tt&gt; metric is allocated bytes &lt;b&gt;not accounted for&lt;/b&gt; by the WiredTiger cache. Even though the increase by the ~7GB is significantly less than the WiredTiger cache (35GB), this still indicates that the memory growth is outside of the WiredTiger cache. The &lt;tt&gt;open cursors&lt;/tt&gt; metric curves strongly suggests cursors are responsible for the increase in the &lt;tt&gt;tcmalloc derived allocated minus wt cache&lt;/tt&gt;.&lt;/li&gt;
	&lt;li&gt;At B, we can see that most of the processes (~126) are blocked under &lt;tt&gt;system processes running&lt;/tt&gt; metric and the &lt;tt&gt;kernel CPU&lt;/tt&gt; is spiking above 60%. The kernel time increase is most probably due to the new threads being created as connections increase.&lt;/li&gt;
	&lt;li&gt;There is some increase in all the IO related metrics between A and B and a significant increase after B: &lt;tt&gt;CPU IO wait&lt;/tt&gt;, &lt;tt&gt;disk Utilization&lt;/tt&gt;, &lt;tt&gt;disk xvdc bytes read&lt;/tt&gt; and &lt;tt&gt;xvdc read requests issued&lt;/tt&gt;. The &lt;tt&gt;queue length&lt;/tt&gt; is ~proportional to &lt;tt&gt;IO wait&lt;/tt&gt;, which means that the CPU resources are just waiting for I/O access which usually indicates that the IO subsystem just being too slow/busy to handle the incoming requests.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&lt;b&gt;Evidence of a shift in the workload&lt;/b&gt;:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/162605/162605_workload_shift.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;By inspecting the IO related metrics, we can see that at A, there is a change in the behavior between &lt;tt&gt;wt block-manager bytes read&lt;/tt&gt; and the &lt;tt&gt;disk xvdc bytes read&lt;/tt&gt;. Before the event, the &lt;tt&gt;wt block-manager bytes read&lt;/tt&gt; metric is ~500 MB/s, yet, the &lt;tt&gt;disk xvdc bytes read&lt;/tt&gt; metric is only ~10 MB/s.&lt;/li&gt;
	&lt;li&gt;When the incident starts at A (and as it progresses) this ratio changes. This strongly indicates that the workload must have shifted to a different part of the dataset &lt;b&gt;that is not in FS cache&lt;/b&gt; and from this point, we are reading mainly from disk.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;To prevent this from happening again in the future, I recommend keeping your dataset in the cache.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dima&lt;/p&gt;




</comment>
                            <comment id="1639441" author="tch@virtall.com" created="Thu, 3 Aug 2017 07:03:05 +0000"  >&lt;p&gt;We were overloaded when mongod crashed (as you can only see from &quot;only configured to support 20020 sessions&quot; - mongod was slow, and new connections were being made, while the old ones were not yet served).&lt;/p&gt;

&lt;p&gt;Just not sure if it was a result of mongo malfunctioning, or something else - we never had a similar issue before.&lt;/p&gt;


&lt;p&gt;So after we&apos;ve realised it crashed, we&apos;ve stopped the instance, then started as a bigger one.&lt;/p&gt;


&lt;p&gt;Ins syslog, there isn&apos;t really anything interesting - DHCP renewals and such. There aren&apos;t any OOM, crashes, nothing unexpected.&lt;/p&gt;</comment>
                            <comment id="1639439" author="dmitry.agranat" created="Thu, 3 Aug 2017 06:45:17 +0000"  >&lt;p&gt;Thanks for uploading the &lt;tt&gt;$dbpath/diagnostic.data&lt;/tt&gt; directory &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tch%40virtall.com&quot; class=&quot;user-hover&quot; rel=&quot;tch@virtall.com&quot;&gt;tch@virtall.com&lt;/a&gt;. I am still looking at the provided data to understand exactly what happened, but in the meantime, can you clarify one thing?&lt;/p&gt;

&lt;p&gt;At the time of the incident, at &lt;tt&gt;2017-07-31T04:29:57&lt;/tt&gt;, the server had 8 CPU and 60GB RAM (if it&apos;s AWS Instance than it&apos;s i2.2xlarge). 3 minutes later, at &lt;tt&gt;2017-07-31T04:33:00&lt;/tt&gt; I can see that the server had 16 CPU and 122GB RAM (if it&apos;s AWS Instance than it&apos;s i2.4xlarge). Could you clarify if you have manually changed the Instance type and if yes, was it intentional?&lt;/p&gt;

&lt;p&gt;Also, if you could attach the &lt;tt&gt;syslog&lt;/tt&gt; from this server that contains the crash that could be helpful.&lt;/p&gt;</comment>
                            <comment id="1637006" author="tch@virtall.com" created="Mon, 31 Jul 2017 22:48:30 +0000"  >&lt;p&gt;The log contains some info which is not public (we had slow log enabled).&lt;/p&gt;

&lt;p&gt;$dbpath/diagnostic.data: &lt;a href=&quot;https://www.virtall.com/tmp/diagnostic.data.tar.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://www.virtall.com/tmp/diagnostic.data.tar.gz&lt;/a&gt;&lt;/p&gt;
</comment>
                            <comment id="1636394" author="bruce.lucas@10gen.com" created="Mon, 31 Jul 2017 15:45:30 +0000"  >&lt;p&gt;Hi Tomasz,&lt;/p&gt;

&lt;p&gt;Can you please archive and attach the $dbpath/diagnostic.data directory?&lt;/p&gt;

&lt;p&gt;Also, if you could attach the complete mongod log file that contains the crash that could be helpful.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Bruce&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="411189">SERVER-30462</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="162606" name="overall_event.png" size="459498" author="dmitry.agranat@mongodb.com" created="Sun, 6 Aug 2017 15:52:59 +0000"/>
                            <attachment id="162605" name="workload_shift.png" size="173284" author="dmitry.agranat@mongodb.com" created="Sun, 6 Aug 2017 15:52:44 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 31 Jul 2017 15:45:30 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 27 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>backlog-server-pm</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 27 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>dmitry.agranat@mongodb.com</customfieldvalue>
            <customfieldvalue>tch@virtall.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htbydj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|ht3r6n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[kelsey.schubert@mongodb.com]]></customfieldvalue>
        <customfieldvalue><![CDATA[dmitry.agranat@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htbkg7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>