<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:02:30 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-3236] High CPU wait load on capped collection reaching it&apos;s limit</title>
                <link>https://jira.mongodb.org/browse/SERVER-3236</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;I have very strange behavior of mongo on out Nimble server. We use mongodb to run cache for our product. Cache consists of single capped collection with few indexed fields. Here&apos;s the stats for it: &lt;/p&gt;

&lt;p&gt;db&amp;gt; db.cache.stats() &lt;br/&gt;
{ &lt;br/&gt;
        &quot;ns&quot; : &quot;nimble_cache.cache&quot;, &lt;br/&gt;
        &quot;count&quot; : 6795240, &lt;br/&gt;
        &quot;size&quot; : 29891215048, &lt;br/&gt;
        &quot;avgObjSize&quot; : 4398.8461111013, &lt;br/&gt;
        &quot;storageSize&quot; : 30000000304, &lt;br/&gt;
        &quot;numExtents&quot; : 15, &lt;br/&gt;
        &quot;nindexes&quot; : 3, &lt;br/&gt;
        &quot;lastExtentSize&quot; : 256, &lt;br/&gt;
        &quot;paddingFactor&quot; : 1, &lt;br/&gt;
        &quot;flags&quot; : 0, &lt;br/&gt;
        &quot;totalIndexSize&quot; : 3472837440, &lt;br/&gt;
        &quot;indexSizes&quot; : &lt;/p&gt;
{ 
                &quot;cache_id_-1_created_-1&quot; : 561824704, 
                &quot;depends_on_1_deadline_-1&quot; : 2351080384, 
                &quot;cache_id_1_resource_1_created_-1&quot; : 559932352 
        }
&lt;p&gt;, &lt;br/&gt;
        &quot;capped&quot; : 1, &lt;br/&gt;
        &quot;max&quot; : 2147483647, &lt;br/&gt;
        &quot;ok&quot; : 1 &lt;/p&gt;


&lt;p&gt;} &lt;/p&gt;

&lt;p&gt;It runs on m2.2xlarge EC2 instance that have 34 GBs of memory. Here&apos;s &lt;br/&gt;
memory stats of the mongo instance: &lt;/p&gt;

&lt;p&gt;&amp;gt; db.serverStatus().mem &lt;/p&gt;

{ 
        &quot;bits&quot; : 64, 
        &quot;resident&quot; : 6300, 
        &quot;virtual&quot; : 34749, 
        &quot;supported&quot; : true, 
        &quot;mapped&quot; : 32960 

} 

&lt;p&gt;The load on server isn&apos;t really high (lock percentage is terrible however): &lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@ip-10-164-1-195 ~&amp;#93;&lt;/span&gt;# /opt/mongo/bin/mongostat &lt;br/&gt;
connected to: 127.0.0.1 &lt;br/&gt;
insert  query update delete getmore command flushes mapped  vsize res faults locked % idx miss %     qr|qw   ar|aw  netIn netOut conn       time &lt;br/&gt;
    93    368     89      0       0      86       0  32.2g  33.9g 6.2g      7     39.8          0       5|7     5|8   444k   684k 151   22:56:44 &lt;br/&gt;
    66    302     85      0       0      90       0  32.2g  33.9g 6.2g      3     77.4          0       0|0     0|0   325k   504k 151   22:56:45 &lt;br/&gt;
    47    153     43      0       0      44       0  32.2g  33.9g 6.2g      4     77.1          0       0|0     0|0   327k   386k 151   22:56:46 &lt;br/&gt;
    67    256     70      0       0      66       0  32.2g  33.9g 6.2g      4     35.9          0       6|8     6|9   329k   437k 151   22:56:47 &lt;br/&gt;
   103    384    104      0       0     106       0  32.2g  33.9g 6.2g      5     60.9          0       6|6     6|7   449k   755k 151   22:56:48 &lt;br/&gt;
    56    207     71      0       0      75       0  32.2g  33.9g 6.2g      5     53.3          0       5|6     5|7   390k   623k 151   22:56:49 &lt;br/&gt;
    47    201     40      0       0      39       0  32.2g  33.9g 6.2g      4     78.1          0       4|4     4|5   348k   473k 151   22:56:50 &lt;br/&gt;
    99    381    117      0       0     121       0  32.2g  33.9g 6.2g      5     27.6          0       0|0     0|0   445k   766k 151   22:56:51 &lt;br/&gt;
    78    308     89      0       0      90       0  32.2g  33.9g 6.21g     5     39.3          0       0|0     0|0   461k   657k 151   22:56:52 &lt;br/&gt;
   115    374    122      0       0     120       0  32.2g  33.9g 6.21g     8      4.7          0       3|7     3|8   484k   647k 151   22:56:54 &lt;/p&gt;

&lt;p&gt;All updates are on non-indexed fields. All queries are using indexes. &lt;/p&gt;

&lt;p&gt;The problem I have is very high disk IO activity. That causes that locks level above. Here&apos;s the output of iostat: &lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;root@ip-10-164-1-195 ~&amp;#93;&lt;/span&gt;# iostat -xdm /dev/sdk 2 &lt;br/&gt;
Linux 2.6.21.7-2.fc8xen (ip-10-164-1-195)       06/07/11 &lt;/p&gt;

&lt;p&gt;Device:         rrqm/s   wrqm/s   r/s   w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await  svctm  %util &lt;br/&gt;
sdk               0.05   259.76 83.07 407.36     2.52     5.83 34.86    43.29   88.28   1.63  79.93 &lt;br/&gt;
sdk               0.00     0.00  6.00 437.00     0.08     1.74  8.42   105.36  260.29   2.26  99.90 &lt;br/&gt;
sdk               0.00     0.00  6.00 741.00     0.06     3.03  8.47    39.87   67.59   1.32  98.95 &lt;br/&gt;
sdk               0.00     4.00 25.00 2519.00    0.17    10.07  8.24    80.18   31.24   0.39  98.30 &lt;br/&gt;
sdk               0.00    25.00 14.00 1530.00    0.14     6.33  8.59    23.92   15.76   0.64  98.10 &lt;br/&gt;
sdk               0.00     2.50  6.50 422.50     0.07     1.73  8.57    55.19   96.98   2.32  99.60 &lt;br/&gt;
sdk               0.00     0.50 11.50 135.00     0.11     0.54  9.17    44.41  363.53   6.82  99.95 &lt;br/&gt;
sdk               0.00     0.00  8.00 104.00     0.08     0.41  8.89   108.89  617.02   8.93 100.05 &lt;br/&gt;
sdk               0.00     0.00 14.50 270.50     0.13     1.08  8.65    69.87  390.25   3.51 100.00 &lt;br/&gt;
sdk               0.00     0.00  6.00 154.00     0.05     0.61  8.45    51.68  305.67   6.25 100.05 &lt;br/&gt;
sdk               0.00     0.00  9.50 204.50     0.07     0.86  8.93    24.03  139.49   4.67  99.90 &lt;/p&gt;

&lt;p&gt;I have trying disabling journaling or increasing syncdelay parameter (now it&apos;s set to 300 seconds). No positive results so far. The one thing I mentioned is that after mongo restart everything works pretty good for about ~10-15 minutes. However after that period of time disk IO starts raising and in few minutes device utilization reaches 100%. &lt;/p&gt;

&lt;p&gt;I&apos;m really out of ideas of how to deal with it. &lt;/p&gt;</description>
                <environment>Linux ip-10-164-19-105 2.6.21.7-2.fc8xen #1 SMP Fri Feb 15 12:34:28 EST 2008 x86_64 x86_64 x86_64 GNU/Linux&lt;br/&gt;
&lt;br/&gt;
EC2 m2.2xlarge instance</environment>
        <key id="18090">SERVER-3236</key>
            <summary>High CPU wait load on capped collection reaching it&apos;s limit</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="1" iconUrl="https://jira.mongodb.org/images/icons/priorities/blocker.svg">Blocker - P1</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="scotthernandez">Scott Hernandez</assignee>
                                    <reporter username="rmihael@gmail.com">Michael Korbakov</reporter>
                        <labels>
                            <label>performance</label>
                    </labels>
                <created>Thu, 9 Jun 2011 19:38:52 +0000</created>
                <updated>Tue, 12 Jul 2016 00:19:11 +0000</updated>
                            <resolved>Fri, 2 Sep 2011 14:07:29 +0000</resolved>
                                    <version>1.8.1</version>
                                                    <component>Performance</component>
                                        <votes>0</votes>
                                    <watches>0</watches>
                                                                                                                <comments>
                            <comment id="52227" author="eliot" created="Fri, 2 Sep 2011 14:07:29 +0000"  >&lt;p&gt;Thanks&lt;/p&gt;</comment>
                            <comment id="52226" author="m.korbakov@nimble.com" created="Fri, 2 Sep 2011 14:04:03 +0000"  >&lt;p&gt;The problem turned out to be in EBS snapshots that we&apos;re using for backups. This cache collection was under heavy updates and snapshotting is turning some heavy IO load on it. We solved the problem by reducing the frequency of snapshots.&lt;/p&gt;</comment>
                            <comment id="51378" author="eliot" created="Tue, 30 Aug 2011 01:42:27 +0000"  >&lt;p&gt;Any update from your side?&lt;/p&gt;</comment>
                            <comment id="37047" author="rmihael@gmail.com" created="Fri, 10 Jun 2011 16:36:39 +0000"  >&lt;p&gt;No there&apos;s no any other collections on this database.&lt;br/&gt;
There&apos;s single kind of updates running on this collection, query part is using index, update part is $set on non-indexed field.&lt;/p&gt;

&lt;p&gt;We already tried profiling. There&apos;s three different queries run on this collection: insert of new cache record, invalidation of cache record and fetching cache record. Query of each types were reported as &quot;slow&quot; by profiling at different times.&lt;/p&gt;

&lt;p&gt;One thing that I&apos;ve forgot to mention in the initial problem description: we&apos;re running backups using EBS snapshots. We don&apos;t stop Mongo server and do this snapshot &quot;on the fly&quot;, relying on journaling for recovery. May be these backups contribute something to the problem.&lt;/p&gt;

&lt;p&gt;Unfortunately I can&apos;t get actual mongostat/iostat numbers now &amp;#8211; we&apos;re experimenting by ourselves trying to solve this issue and current configuration isn&apos;t the one described above. We&apos;ll get back to original configuration with capped collection after weekends, I&apos;ll attach this stats then.&lt;/p&gt;</comment>
                            <comment id="36942" author="scotthernandez" created="Thu, 9 Jun 2011 22:42:42 +0000"  >&lt;p&gt;Aside from the capped collection are there other collections which are on the same servers? What updates are you doing?&lt;/p&gt;

&lt;p&gt;Can you turn on database profiling and report on the some of the slowest operations which you see? &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Database+Profiler&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Database+Profiler&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Also, can you provide a few minutes of mongostat/iostat numbers (as an attachment)?&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 9 Jun 2011 20:02:44 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        12 years, 24 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            12 years, 24 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>eliot</customfieldvalue>
            <customfieldvalue>m.korbakov@nimble.com</customfieldvalue>
            <customfieldvalue>rmihael@gmail.com</customfieldvalue>
            <customfieldvalue>scotthernandez</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hroy3z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrifxr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>21234</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht0c0v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>