<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:22:44 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-30114] Monitor cumulative time spent in tcmalloc spin lock in serverStatus</title>
                <link>https://jira.mongodb.org/browse/SERVER-30114</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We sometimes encounter allocator bottlenecks that currently require asking users to collect stack traces with gdb to diagnose. We could diagnose these bottlenecks from FTDC data if we counted cumulative time spent waiting for the primary lock involved. Typical stacks for such a bottleneck look like this:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#0  0x00000000014a6f85 in base::internal::SpinLockDelay(int volatile*, int, int) ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#1  0x00000000014a6e57 in SpinLock::SlowLock() ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#2  0x00000000014a9393 in tcmalloc::CentralFreeList::RemoveRange(void**, void**, int) ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#3  0x00000000014b4d2a in tcmalloc::ThreadCache::FetchFromCentralCache(unsigned long, unsigned long) ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#0  0x00000000014a6f85 in base::internal::SpinLockDelay(int volatile*, int, int) ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#1  0x00000000014a6e57 in SpinLock::SlowLock() ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#2  0x00000000014a8efd in tcmalloc::CentralFreeList::InsertRange(void*, void*, int) ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#3  0x00000000014b4ed8 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#4  0x00000000014b4f7d in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) ()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="403207">SERVER-30114</key>
            <summary>Monitor cumulative time spent in tcmalloc spin lock in serverStatus</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="mark.benvenuto@mongodb.com">Mark Benvenuto</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                            <label>SWDI</label>
                    </labels>
                <created>Wed, 12 Jul 2017 18:31:17 +0000</created>
                <updated>Mon, 30 Oct 2023 23:15:17 +0000</updated>
                            <resolved>Fri, 9 Feb 2018 16:19:01 +0000</resolved>
                                                    <fixVersion>3.2.20</fixVersion>
                    <fixVersion>3.4.14</fixVersion>
                    <fixVersion>3.6.4</fixVersion>
                    <fixVersion>3.7.2</fixVersion>
                                    <component>Diagnostics</component>
                                        <votes>25</votes>
                                    <watches>28</watches>
                                                                                                                <comments>
                            <comment id="1816224" author="xgen-internal-githook" created="Mon, 26 Feb 2018 18:07:17 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;mark.benvenuto@mongodb.com&apos;, &apos;name&apos;: &apos;Mark Benvenuto&apos;, &apos;username&apos;: &apos;markbenvenuto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30114&quot; title=&quot;Monitor cumulative time spent in tcmalloc spin lock in serverStatus&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30114&quot;&gt;&lt;del&gt;SERVER-30114&lt;/del&gt;&lt;/a&gt; Add simple slow spinlock monitoring&lt;/p&gt;

&lt;p&gt;(cherry picked from commit ba957823288d674f34c8345059f35f34cb4dd055)&lt;br/&gt;
Branch: v3.6&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/35df11d74e04899a01069951fe3bad9a8295859c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/35df11d74e04899a01069951fe3bad9a8295859c&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1810815" author="xgen-internal-githook" created="Tue, 20 Feb 2018 20:18:57 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;mark.benvenuto@mongodb.com&apos;, &apos;name&apos;: &apos;Mark Benvenuto&apos;, &apos;username&apos;: &apos;markbenvenuto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30114&quot; title=&quot;Monitor cumulative time spent in tcmalloc spin lock in serverStatus&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30114&quot;&gt;&lt;del&gt;SERVER-30114&lt;/del&gt;&lt;/a&gt; Add simple slow spinlock monitoring&lt;/p&gt;

&lt;p&gt;(cherry picked from commit ba957823288d674f34c8345059f35f34cb4dd055)&lt;br/&gt;
Branch: v3.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/bcf597ce00b63a04703e6a96414b5fab85e86d34&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/bcf597ce00b63a04703e6a96414b5fab85e86d34&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1810294" author="xgen-internal-githook" created="Tue, 20 Feb 2018 16:25:56 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;mark.benvenuto@mongodb.com&apos;, &apos;name&apos;: &apos;Mark Benvenuto&apos;, &apos;username&apos;: &apos;markbenvenuto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30114&quot; title=&quot;Monitor cumulative time spent in tcmalloc spin lock in serverStatus&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30114&quot;&gt;&lt;del&gt;SERVER-30114&lt;/del&gt;&lt;/a&gt; Add simple slow spinlock monitoring&lt;/p&gt;

&lt;p&gt;(cherry picked from commit ba957823288d674f34c8345059f35f34cb4dd055)&lt;br/&gt;
Branch: v3.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/f8969ad5eda7a5e3b753d01fa1861b645425c7e6&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/f8969ad5eda7a5e3b753d01fa1861b645425c7e6&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1800896" author="xgen-internal-githook" created="Fri, 9 Feb 2018 16:18:19 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;mark.benvenuto@mongodb.com&apos;, &apos;name&apos;: &apos;Mark Benvenuto&apos;, &apos;username&apos;: &apos;markbenvenuto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30114&quot; title=&quot;Monitor cumulative time spent in tcmalloc spin lock in serverStatus&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30114&quot;&gt;&lt;del&gt;SERVER-30114&lt;/del&gt;&lt;/a&gt; Add simple slow spinlock monitoring&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/ba957823288d674f34c8345059f35f34cb4dd055&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/ba957823288d674f34c8345059f35f34cb4dd055&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1783636" author="bruce.lucas@10gen.com" created="Wed, 24 Jan 2018 17:05:31 +0000"  >&lt;p&gt;Once we introduce the metric we can see whether the workloads are generating significant tcmalloc spinlock contention, and if they are then we know they are useful tests for gauging any impact of introducing the metric.&lt;/p&gt;</comment>
                            <comment id="1783619" author="acm" created="Wed, 24 Jan 2018 16:57:08 +0000"  >&lt;p&gt;I meant it the other way around - I want to use those workloads to ensure that introducing the metric has acceptable (hopefully negligible) performance overhead before we propose it upstream and/or merge it ourselves.&lt;/p&gt;</comment>
                            <comment id="1783611" author="bruce.lucas@10gen.com" created="Wed, 24 Jan 2018 16:49:56 +0000"  >&lt;p&gt;Once we&apos;ve introduce the metric we can check those workloads to see if they&apos;re generating a signal.&lt;/p&gt;</comment>
                            <comment id="1783377" author="david.daly" created="Wed, 24 Jan 2018 14:45:02 +0000"  >&lt;p&gt;We&apos;ve seen interesting tcmalloc related behavior with the mixed_* workloads in core_workloads in the past. It has a large number of threads, with groups of threads doing different simple operations (find, update, delete, ...)&lt;/p&gt;</comment>
                            <comment id="1782548" author="bruce.lucas@10gen.com" created="Tue, 23 Jan 2018 18:57:21 +0000"  >&lt;p&gt;Simple performance test. Java load generator (attached) run as follows:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;java -cp ... load host=localhost op=spinlock threads=200 tps=1e100 qps=1e100 dur=30 size=0 batch=1000&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;This is 200 threads all inserting tiny (_id-only) documents in batches of 1000. Standalone, no journal, no checkpoints, running on 12 cores. Alternating 30-second runs with and without spinlock monitor, restarting mongod each run:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/176623/176623_run3.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;As the spinlock monitor stats show we&apos;re hitting the spinlock moderately hard because we have 200 threads on 12 cores filling the cache with tiny documents; on average at all times we have 5-10 threads waiting for the spinlock.&lt;/p&gt;

&lt;p&gt;No visible difference. Numerically, total number of inserts in each 30 second run:&lt;/p&gt;

&lt;p&gt;with spinlock monitor:  21157000 20973000 21047000 21267000 21373000 20563000 21208000 20299000 20714000 21007000&lt;/p&gt;

&lt;p&gt;without spinlock monitor:  20899000 21677000 20777000 21190000 20996000 21061000 21197000 20857000 21705000 21330000&lt;/p&gt;

&lt;p&gt;With spinlock monitor average is about 1% less than without, which is about 0.6 standard deviations, so in the noise.&lt;/p&gt;</comment>
                            <comment id="1758611" author="bruce.lucas@10gen.com" created="Fri, 22 Dec 2017 15:10:48 +0000"  >&lt;p&gt;I&apos;ve attached a new POC (spinlock2.diff) that measures the actual time spent in SpinLockDelay rather than just accumulating the supplied sleep times because the Linux design uses futex with wakeups so it may wake up before the supplied sleep time elapses. This requires getting the current time, but since this is only done if we also do an expensive delay I think it&apos;s unlikely to have a performance impact.&lt;/p&gt;</comment>
                            <comment id="1757921" author="bruce.lucas@10gen.com" created="Thu, 21 Dec 2017 16:39:09 +0000"  >&lt;p&gt;I think that we can get 90% of the diagnostic value with something much simpler&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;monitor only time spent in SpinLockDelay, as that is where I&apos;ve seen most of the time spent when there is tcmalloc contention.&lt;/li&gt;
	&lt;li&gt;keep only a single counter for all tcmalloc spinlocks; this will allow us to determine when tcmalloc is the bottleneck.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;I&apos;ve attached a POC implementation; it&apos;s only a few lines of code.&lt;/p&gt;

&lt;p&gt;Here&apos;s an example in use:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/174169/174169_spinlock.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Left is the current default executor; right is the adaptive executor. We see some allocator contention in the first run, although it maxes out at about 37 threads out of 145 so it&apos;s probably not impacting performance much.&lt;/p&gt;

&lt;p&gt;Would it be feasible to put this monitoring in?&lt;/p&gt;</comment>
                            <comment id="1754805" author="mark.benvenuto" created="Mon, 18 Dec 2017 18:56:01 +0000"  >&lt;p&gt;There are a few reasons why I do not want to add stats tracking on a permanent basis to MongoDB:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;It is expensive to maintain the counters at runtime. In my simple naive prototype, I think it was a 15-20% hit on performance to run with the counters. In any final solution, it adds a lot of contention to cache lines to maintain counters for each spinlock, or a more complicated solution needs to be implemented that avoids heavy cache line contention.&lt;/li&gt;
	&lt;li&gt;It will be expensive to maintain going forward. The prototype I linked to was ~400 changed lines, and did not even support debug tcmalloc. In order to track spin lock statistics accurately, we need to ensure every call site is updated. The prototype is one way to do that, and since it affects every call site, it is is an expensive change to maintain across TCMalloc versions.&lt;/li&gt;
	&lt;li&gt;There are a lot of counters to report information on. We would likely only ask customers to collect this information in FTDC post a problem. In retail tcmalloc, there are approximately 99 counters to log to FTDC. This includes the 88 individual counters for the CentralFreeList.&lt;/li&gt;
&lt;/ol&gt;
</comment>
                            <comment id="1730242" author="bruce.lucas@10gen.com" created="Mon, 20 Nov 2017 14:15:22 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=mark.benvenuto&quot; class=&quot;user-hover&quot; rel=&quot;mark.benvenuto&quot;&gt;mark.benvenuto&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=acm&quot; class=&quot;user-hover&quot; rel=&quot;acm&quot;&gt;acm&lt;/a&gt;, my apologies, but I think I neglected to bring this ticket up when we were reviewing tcmalloc work for 3.6. My concern is that we made a fairly major change in the way we use tcmalloc (disabling markThreadIdle), with an option to re-enable it if a particular customer workload proves to need it, but we don&apos;t have a good way to monitor whether tcmalloc has become a bottleneck. In my experience with customer issues typically tcmalloc bottlenecks here but leaves no trace in the metrics, making it difficult to diagnose. This metric would fix that.&lt;/p&gt;

&lt;p&gt;I realize that it is probably too tight to get this into 3.6.0, particularly since we need to make sure this monitoring doesn&apos;t introduce performance problems itself, but would it be possible to do this early in 3.7 and backport it to 3.6.small?&lt;/p&gt;</comment>
                            <comment id="1620302" author="mark.benvenuto" created="Wed, 12 Jul 2017 18:51:44 +0000"  >&lt;p&gt;Prototype code that tracks tcmalloc SpinLock spinning by spinlock type:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://github.com/markbenvenuto/mongo/commit/eda17993ec4638d38dc2c8d0e105082219c124e5&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/markbenvenuto/mongo/commit/eda17993ec4638d38dc2c8d0e105082219c124e5&lt;/a&gt;&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="176624" name="load.java" size="15776" author="bruce.lucas@mongodb.com" created="Tue, 23 Jan 2018 18:58:17 +0000"/>
                            <attachment id="176623" name="run3.png" size="145325" author="bruce.lucas@mongodb.com" created="Tue, 23 Jan 2018 18:56:23 +0000"/>
                            <attachment id="174170" name="spinlock.diff" size="3427" author="bruce.lucas@mongodb.com" created="Thu, 21 Dec 2017 16:37:52 +0000"/>
                            <attachment id="174169" name="spinlock.png" size="101430" author="bruce.lucas@mongodb.com" created="Thu, 21 Dec 2017 16:37:45 +0000"/>
                            <attachment id="174225" name="spinlock2.diff" size="3320" author="bruce.lucas@mongodb.com" created="Fri, 22 Dec 2017 15:07:01 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="15141"><![CDATA[v3.6]]></customfieldvalue>
    <customfieldvalue key="14340"><![CDATA[v3.4]]></customfieldvalue>
    <customfieldvalue key="13440"><![CDATA[v3.2]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000ZqUlIIAV]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 12 Jul 2017 18:51:44 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 50 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-863</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 50 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>andrew.morrow@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>david.daly@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>mark.benvenuto@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htarvj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr9xtz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2093">Platforms 2018-01-29</customfieldvalue>
    <customfieldvalue id="2129">Platforms 2018-02-12</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11861" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>User Summary</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="11856"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htady7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>