<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:06:27 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-4597] Full rsync: build index in background</title>
                <link>https://jira.mongodb.org/browse/SERVER-4597</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Hello,&lt;/p&gt;

&lt;p&gt;Environnement: replica set 1primary(1.8.3) + 1secondary(2.0.2) + 1arbiter.&lt;/p&gt;

&lt;p&gt;Secondary has been offline (RS102) and is now too far behind to catch up, I will need to resync.&lt;br/&gt;
I perform full resync, but always crashed when build index.&lt;br/&gt;
I have 200 000 000 docs on my collection and 3index. Crash appear on second index&apos;s building.&lt;br/&gt;
I try to increase oplog&apos;size and make rsync during low primary activity, no success ...&lt;/p&gt;

&lt;p&gt;it&apos;s possible to build index in background? someone have an idea/solutions ?&lt;/p&gt;

&lt;p&gt;sorry for my in english, i try to improve it &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;br/&gt;
Kind regard,&lt;/p&gt;
</description>
                <environment></environment>
        <key id="27596">SERVER-4597</key>
            <summary>Full rsync: build index in background</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="1" iconUrl="https://jira.mongodb.org/images/icons/priorities/blocker.svg">Blocker - P1</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="5">Cannot Reproduce</resolution>
                                        <assignee username="dan@mongodb.com">Daniel Pasette</assignee>
                                    <reporter username="lujeni">Julien Thebault</reporter>
                        <labels>
                    </labels>
                <created>Tue, 3 Jan 2012 16:18:15 +0000</created>
                <updated>Wed, 20 Jun 2012 18:11:11 +0000</updated>
                            <resolved>Sat, 25 Feb 2012 05:31:01 +0000</resolved>
                                    <version>1.8.3</version>
                    <version>2.0.2</version>
                                                    <component>Index Maintenance</component>
                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>2</watches>
                                                                                                                <comments>
                            <comment id="134901" author="cb_john" created="Wed, 20 Jun 2012 18:11:11 +0000"  >&lt;p&gt;We had what seemed to be the same issue over here with our servers at chartboost. It&apos;s preventing us from successfully adding any new nodes to the replica sets in our main mongo sharded (4 shards) cluster... the team here is evaluating moving to a different db platform but I&apos;m hoping that there&apos;s a patch or workaround that will allow us to continue growing the cluster without needing to boot up new boxes with double the RAM just to successfully replicate.&lt;/p&gt;

&lt;p&gt;From a new box (dual Core 8gb vm instance on Joyentcloud running centos 6), we installed the latest mongo 2.0.5 and started replication for the shard... the external sort finished but then mongo eventually crashed... here&apos;s the end of the mongo log from /var/log/mongo/mongod.log . &lt;/p&gt;

&lt;p&gt;(Note how fast it was going initially and then how slow it got near the end. took 23 minutes to get to 80% and then 3 hours or so to get from 80% to 89% before it crashed.) &lt;/p&gt;

&lt;p&gt;Wed Jun 20 02:04:51 &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt;     external sort used : 113 files  in 728 secs&lt;br/&gt;
                750000/111903891        0%&lt;br/&gt;
                1497600/111903891       1%&lt;br/&gt;
                2312200/111903891       2%&lt;br/&gt;
                3064100/111903891       2%&lt;br/&gt;
                3844600/111903891       3%&lt;br/&gt;
                4562100/111903891       4%&lt;br/&gt;
                5265400/111903891       4%&lt;br/&gt;
                5999500/111903891       5%&lt;br/&gt;
                6776200/111903891       6%&lt;br/&gt;
                7537600/111903891       6%&lt;br/&gt;
                8248300/111903891       7%&lt;br/&gt;
                9042000/111903891       8%&lt;br/&gt;
                9840000/111903891       8%&lt;br/&gt;
                10581600/111903891      9%&lt;br/&gt;
                11316200/111903891      10%&lt;br/&gt;
                12021600/111903891      10%&lt;br/&gt;
                12783200/111903891      11%&lt;br/&gt;
                13545800/111903891      12%&lt;br/&gt;
                14290000/111903891      12%&lt;br/&gt;
                15025900/111903891      13%&lt;br/&gt;
                15714800/111903891      14%&lt;br/&gt;
                16569400/111903891      14%&lt;br/&gt;
                17312500/111903891      15%&lt;br/&gt;
                18118200/111903891      16%&lt;br/&gt;
                19574500/111903891      17%&lt;br/&gt;
                20338300/111903891      18%&lt;br/&gt;
                20966200/111903891      18%&lt;br/&gt;
                21679500/111903891      19%&lt;br/&gt;
                22386000/111903891      20%&lt;br/&gt;
Wed Jun 20 02:09:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /data/mongo/chartboost.23, filling with zeroes...&lt;br/&gt;
Wed Jun 20 02:09:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /data/mongo/chartboost.23, size: 2047MB,  took 0.022 secs&lt;br/&gt;
                22987800/111903891      20%&lt;br/&gt;
                23791400/111903891      21%&lt;br/&gt;
                24548300/111903891      21%&lt;br/&gt;
                25323700/111903891      22%&lt;br/&gt;
                26097700/111903891      23%&lt;br/&gt;
                26826700/111903891      23%&lt;br/&gt;
                27597400/111903891      24%&lt;br/&gt;
                28278800/111903891      25%&lt;br/&gt;
                28989300/111903891      25%&lt;br/&gt;
                29672100/111903891      26%&lt;br/&gt;
                30367600/111903891      27%&lt;br/&gt;
                31128000/111903891      27%&lt;br/&gt;
                31857600/111903891      28%&lt;br/&gt;
                32678400/111903891      29%&lt;br/&gt;
                33371000/111903891      29%&lt;br/&gt;
                34081400/111903891      30%&lt;br/&gt;
                34828300/111903891      31%&lt;br/&gt;
                35520100/111903891      31%&lt;br/&gt;
                36235000/111903891      32%&lt;br/&gt;
                37028900/111903891      33%&lt;br/&gt;
                37828200/111903891      33%&lt;br/&gt;
                38508000/111903891      34%&lt;br/&gt;
                39263700/111903891      35%&lt;br/&gt;
                40103300/111903891      35%&lt;br/&gt;
Wed Jun 20 02:13:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
                40792200/111903891      36%&lt;br/&gt;
Wed Jun 20 02:14:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
Wed Jun 20 02:14:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
                41561700/111903891      37%&lt;br/&gt;
                42327600/111903891      37%&lt;br/&gt;
Wed Jun 20 02:14:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; old journal file will be removed: /data/mongo/journal/j._19&lt;br/&gt;
                43119100/111903891      38%&lt;br/&gt;
                43918300/111903891      39%&lt;br/&gt;
                44654100/111903891      39%&lt;br/&gt;
                45450900/111903891      40%&lt;br/&gt;
                46157500/111903891      41%&lt;br/&gt;
                46893000/111903891      41%&lt;br/&gt;
Wed Jun 20 02:15:25 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /data/mongo/chartboost.24, filling with zeroes...&lt;br/&gt;
Wed Jun 20 02:15:25 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /data/mongo/chartboost.24, size: 2047MB,  took 0.017 secs&lt;br/&gt;
                47653800/111903891      42%&lt;br/&gt;
                48332700/111903891      43%&lt;br/&gt;
                48959600/111903891      43%&lt;br/&gt;
               49679300/111903891      44%&lt;br/&gt;
                50481100/111903891      45%&lt;br/&gt;
                51233700/111903891      45%&lt;br/&gt;
                51964700/111903891      46%&lt;br/&gt;
                52710700/111903891      47%&lt;br/&gt;
                53529000/111903891      47%&lt;br/&gt;
                54276100/111903891      48%&lt;br/&gt;
                54958500/111903891      49%&lt;br/&gt;
                55740700/111903891      49%&lt;br/&gt;
                56435700/111903891      50%&lt;br/&gt;
                57150400/111903891      51%&lt;br/&gt;
                57891900/111903891      51%&lt;br/&gt;
                58603700/111903891      52%&lt;br/&gt;
                59331000/111903891      53%&lt;br/&gt;
                59991700/111903891      53%&lt;br/&gt;
                60643500/111903891      54%&lt;br/&gt;
                61331300/111903891      54%&lt;br/&gt;
                61919100/111903891      55%&lt;br/&gt;
Wed Jun 20 02:18:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                62555700/111903891      55%&lt;br/&gt;
Wed Jun 20 02:19:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
Wed Jun 20 02:19:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
                63141000/111903891      56%&lt;br/&gt;
                63762200/111903891      56%&lt;br/&gt;
                64425700/111903891      57%&lt;br/&gt;
                65007000/111903891      58%&lt;br/&gt;
Wed Jun 20 02:19:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /data/mongo/chartboost.25, filling with zeroes...&lt;br/&gt;
Wed Jun 20 02:19:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /data/mongo/chartboost.25, size: 2047MB,  took 0.039 secs&lt;br/&gt;
                65603700/111903891      58%&lt;br/&gt;
                66170700/111903891      59%&lt;br/&gt;
                66768000/111903891      59%&lt;br/&gt;
                67311700/111903891      60%&lt;br/&gt;
                67961300/111903891      60%&lt;br/&gt;
                68572100/111903891      61%&lt;br/&gt;
                69221800/111903891      61%&lt;br/&gt;
                69678300/111903891      62%&lt;br/&gt;
                70327700/111903891      62%&lt;br/&gt;
                70968700/111903891      63%&lt;br/&gt;
                71560600/111903891      63%&lt;br/&gt;
                72199600/111903891      64%&lt;br/&gt;
                72809500/111903891      65%&lt;br/&gt;
Wed Jun 20 02:21:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; old journal file will be removed: /data/mongo/journal/j._20&lt;br/&gt;
Wed Jun 20 02:21:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; old journal file will be removed: /data/mongo/journal/j._21&lt;br/&gt;
                73429700/111903891      65%&lt;br/&gt;
                74121000/111903891      66%&lt;br/&gt;
                74753000/111903891      66%&lt;br/&gt;
                75357200/111903891      67%&lt;br/&gt;
                75864200/111903891      67%&lt;br/&gt;
                76519400/111903891      68%&lt;br/&gt;
                77118800/111903891      68%&lt;br/&gt;
                77778800/111903891      69%&lt;br/&gt;
                78456300/111903891      70%&lt;br/&gt;
                79073300/111903891      70%&lt;br/&gt;
               79705400/111903891      71%&lt;br/&gt;
                80272500/111903891      71%&lt;br/&gt;
Wed Jun 20 02:23:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
                80812700/111903891      72%&lt;br/&gt;
Wed Jun 20 02:24:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
Wed Jun 20 02:24:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
                81429300/111903891      72%&lt;br/&gt;
                82046700/111903891      73%&lt;br/&gt;
                82644800/111903891      73%&lt;br/&gt;
Wed Jun 20 02:24:31 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /data/mongo/chartboost.26, filling with zeroes...&lt;br/&gt;
Wed Jun 20 02:24:31 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /data/mongo/chartboost.26, size: 2047MB,  took 0.036 secs&lt;br/&gt;
                83297000/111903891      74%&lt;br/&gt;
                83904100/111903891      74%&lt;br/&gt;
                84475200/111903891      75%&lt;br/&gt;
                84989600/111903891      75%&lt;br/&gt;
                85581700/111903891      76%&lt;br/&gt;
                86154700/111903891      76%&lt;br/&gt;
                86727400/111903891      77%&lt;br/&gt;
                87234200/111903891      77%&lt;br/&gt;
                87742600/111903891      78%&lt;br/&gt;
                88299400/111903891      78%&lt;br/&gt;
                88826000/111903891      79%&lt;br/&gt;
                89226400/111903891      79%&lt;br/&gt;
                89829800/111903891      80%&lt;br/&gt;
                90382700/111903891      80%&lt;br/&gt;
Wed Jun 20 02:26:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;PeriodicTask::Runner&amp;#93;&lt;/span&gt; task: WriteBackManager::cleaner took: 8ms&lt;br/&gt;
                90954000/111903891      81%&lt;br/&gt;
                91552200/111903891      81%&lt;br/&gt;
                92062900/111903891      82%&lt;br/&gt;
                92621600/111903891      82%&lt;br/&gt;
                93130900/111903891      83%&lt;br/&gt;
                93660300/111903891      83%&lt;br/&gt;
Wed Jun 20 02:27:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /data/mongo/chartboost.27, filling with zeroes...&lt;br/&gt;
Wed Jun 20 02:27:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /data/mongo/chartboost.27, size: 2047MB,  took 0.022 secs&lt;br/&gt;
                94167700/111903891      84%&lt;br/&gt;
                94625700/111903891      84%&lt;br/&gt;
                95039400/111903891      84%&lt;br/&gt;
                95456400/111903891      85%&lt;br/&gt;
                95830800/111903891      85%&lt;br/&gt;
                96166400/111903891      85%&lt;br/&gt;
Wed Jun 20 02:28:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300024ms&lt;br/&gt;
Wed Jun 20 02:29:00 &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; old journal file will be removed: /data/mongo/journal/j._22&lt;br/&gt;
                96491100/111903891      86%&lt;br/&gt;
Wed Jun 20 02:29:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
Wed Jun 20 02:29:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                96751100/111903891      86%&lt;br/&gt;
                97008200/111903891      86%&lt;br/&gt;
                97280500/111903891      86%&lt;br/&gt;
                97498000/111903891      87%&lt;br/&gt;
                97898500/111903891      87%&lt;br/&gt;
                98290900/111903891      87%&lt;br/&gt;
                98304600/111903891      87%&lt;br/&gt;
                98315900/111903891      87%&lt;br/&gt;
                98318400/111903891      87%&lt;br/&gt;
                98318900/111903891      87%&lt;br/&gt;
                98319500/111903891      87%&lt;br/&gt;
                98361100/111903891      87%&lt;br/&gt;
                98374600/111903891      87%&lt;br/&gt;
                98377200/111903891      87%&lt;br/&gt;
                98385200/111903891      87%&lt;br/&gt;
                98390700/111903891      87%&lt;br/&gt;
                98404000/111903891      87%&lt;br/&gt;
                98424800/111903891      87%&lt;br/&gt;
                98426800/111903891      87%&lt;br/&gt;
                98428300/111903891      87%&lt;br/&gt;
                98430400/111903891      87%&lt;br/&gt;
                98430900/111903891      87%&lt;br/&gt;
                98431400/111903891      87%&lt;br/&gt;
                98432500/111903891      87%&lt;br/&gt;
                98435500/111903891      87%&lt;br/&gt;
                98436500/111903891      87%&lt;br/&gt;
                98438800/111903891      87%&lt;br/&gt;
                98439900/111903891      87%&lt;br/&gt;
Wed Jun 20 02:33:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                98441300/111903891      87%&lt;br/&gt;
Wed Jun 20 02:34:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
Wed Jun 20 02:34:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                98441800/111903891      87%&lt;br/&gt;
                98443100/111903891      87%&lt;br/&gt;
                98444400/111903891      87%&lt;br/&gt;
                98445000/111903891      87%&lt;br/&gt;
                98446300/111903891      87%&lt;br/&gt;
                98448400/111903891      87%&lt;br/&gt;
                98451300/111903891      87%&lt;br/&gt;
                98455600/111903891      87%&lt;br/&gt;
                98458100/111903891      87%&lt;br/&gt;
                98459000/111903891      87%&lt;br/&gt;
                98461100/111903891      87%&lt;br/&gt;
                98461800/111903891      87%&lt;br/&gt;
                98462500/111903891      87%&lt;br/&gt;
                98463900/111903891      87%&lt;br/&gt;
                98464700/111903891      87%&lt;br/&gt;
                98465700/111903891      87%&lt;br/&gt;
                98466700/111903891      87%&lt;br/&gt;
                98467200/111903891      87%&lt;br/&gt;
                98467700/111903891      87%&lt;br/&gt;
                98468200/111903891      87%&lt;br/&gt;
                98468700/111903891      87%&lt;br/&gt;
                98469800/111903891      87%&lt;br/&gt;
                98470300/111903891      87%&lt;br/&gt;
                98470700/111903891      87%&lt;br/&gt;
                98471000/111903891      87%&lt;br/&gt;
                98475600/111903891      88%&lt;br/&gt;
               98477800/111903891      88%&lt;br/&gt;
                98479800/111903891      88%&lt;br/&gt;
Wed Jun 20 02:38:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                98481600/111903891      88%&lt;br/&gt;
Wed Jun 20 02:39:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
Wed Jun 20 02:39:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                98484300/111903891      88%&lt;br/&gt;
                98486100/111903891      88%&lt;br/&gt;
                98487200/111903891      88%&lt;br/&gt;
                98487900/111903891      88%&lt;br/&gt;
                98507100/111903891      88%&lt;br/&gt;
                98510000/111903891      88%&lt;br/&gt;
                98510500/111903891      88%&lt;br/&gt;
                98512100/111903891      88%&lt;br/&gt;
                98513000/111903891      88%&lt;br/&gt;
                98513500/111903891      88%&lt;br/&gt;
                98514300/111903891      88%&lt;br/&gt;
                98514900/111903891      88%&lt;br/&gt;
                98515700/111903891      88%&lt;br/&gt;
                98516200/111903891      88%&lt;br/&gt;
                98516700/111903891      88%&lt;br/&gt;
                98517600/111903891      88%&lt;br/&gt;
                98518800/111903891      88%&lt;br/&gt;
                98519300/111903891      88%&lt;br/&gt;
                98520300/111903891      88%&lt;br/&gt;
                98520800/111903891      88%&lt;br/&gt;
                98521900/111903891      88%&lt;br/&gt;
                98523100/111903891      88%&lt;br/&gt;
                98523700/111903891      88%&lt;br/&gt;
                98525700/111903891      88%&lt;br/&gt;
                98526400/111903891      88%&lt;br/&gt;
                98527100/111903891      88%&lt;br/&gt;
                98527600/111903891      88%&lt;br/&gt;
                98528100/111903891      88%&lt;br/&gt;
Wed Jun 20 02:43:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                98541800/111903891      88%&lt;br/&gt;
Wed Jun 20 02:44:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
Wed Jun 20 02:44:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                98546300/111903891      88%&lt;br/&gt;
                98548400/111903891      88%&lt;br/&gt;
                98549100/111903891      88%&lt;br/&gt;
                98550000/111903891      88%&lt;br/&gt;
                98551300/111903891      88%&lt;br/&gt;
                98552600/111903891      88%&lt;br/&gt;
                98553100/111903891      88%&lt;br/&gt;
                98553600/111903891      88%&lt;br/&gt;
                98554200/111903891      88%&lt;br/&gt;
                98555300/111903891      88%&lt;br/&gt;
                98555800/111903891      88%&lt;br/&gt;
                98556300/111903891      88%&lt;br/&gt;
                98556800/111903891      88%&lt;br/&gt;
                98557400/111903891      88%&lt;br/&gt;
              98558000/111903891      88%&lt;br/&gt;
                98559200/111903891      88%&lt;br/&gt;
                98560200/111903891      88%&lt;br/&gt;
                98562900/111903891      88%&lt;br/&gt;
                98564700/111903891      88%&lt;br/&gt;
                98566300/111903891      88%&lt;br/&gt;
                98567400/111903891      88%&lt;br/&gt;
                98568800/111903891      88%&lt;br/&gt;
                98570100/111903891      88%&lt;br/&gt;
                98570700/111903891      88%&lt;br/&gt;
                98571300/111903891      88%&lt;br/&gt;
                98571700/111903891      88%&lt;br/&gt;
                98572100/111903891      88%&lt;br/&gt;
                98572600/111903891      88%&lt;br/&gt;
Wed Jun 20 02:48:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
Wed Jun 20 02:49:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
Wed Jun 20 02:49:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                98573100/111903891      88%&lt;br/&gt;
                98573500/111903891      88%&lt;br/&gt;
                98573800/111903891      88%&lt;br/&gt;
                98574300/111903891      88%&lt;br/&gt;
                98574800/111903891      88%&lt;br/&gt;
                98575300/111903891      88%&lt;br/&gt;
                98575800/111903891      88%&lt;br/&gt;
                98576400/111903891      88%&lt;br/&gt;
                98578200/111903891      88%&lt;br/&gt;
                98578800/111903891      88%&lt;br/&gt;
                98579200/111903891      88%&lt;br/&gt;
                98579500/111903891      88%&lt;br/&gt;
                98579900/111903891      88%&lt;br/&gt;
                98580200/111903891      88%&lt;br/&gt;
                98581400/111903891      88%&lt;br/&gt;
                98582700/111903891      88%&lt;br/&gt;
                98583600/111903891      88%&lt;br/&gt;
                98584400/111903891      88%&lt;br/&gt;
                98586100/111903891      88%&lt;br/&gt;
                98587000/111903891      88%&lt;br/&gt;
                98587800/111903891      88%&lt;br/&gt;
                98588300/111903891      88%&lt;br/&gt;
                98588900/111903891      88%&lt;br/&gt;
                98590200/111903891      88%&lt;br/&gt;
                98590700/111903891      88%&lt;br/&gt;
                98591300/111903891      88%&lt;br/&gt;
                98593400/111903891      88%&lt;br/&gt;
                98593900/111903891      88%&lt;br/&gt;
Wed Jun 20 02:53:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6f27fb0b7403c30c7021&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300000ms&lt;br/&gt;
                98594400/111903891      88%&lt;br/&gt;
Wed Jun 20 02:54:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fcf6ee0c637de224e7ecb5d&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
Wed Jun 20 02:54:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                98595100/111903891      88%&lt;br/&gt;
                98596100/111903891      88%&lt;br/&gt;
                98597100/111903891      88%&lt;/p&gt;

&lt;p&gt;etc etc... and 4 hours later it&apos;s now up to ... 89%:&lt;/p&gt;

&lt;p&gt;Wed Jun 20 05:39:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn23&amp;#93;&lt;/span&gt; command admin.$cmd command: &lt;/p&gt;
{ writebacklisten: ObjectId(&apos;4fbc4fe47fc06f85f192440a&apos;) }
&lt;p&gt; ntoreturn:1 reslen:44 300001ms&lt;br/&gt;
                99588700/111903891      88%&lt;br/&gt;
                99589900/111903891      88%&lt;br/&gt;
                99590400/111903891      88%&lt;br/&gt;
                99590800/111903891      88%&lt;br/&gt;
                99591800/111903891      88%&lt;br/&gt;
                99592700/111903891      88%&lt;br/&gt;
                99593000/111903891      88%&lt;br/&gt;
                99593700/111903891      88%&lt;br/&gt;
                99594100/111903891      88%&lt;br/&gt;
                99594400/111903891      88%&lt;br/&gt;
                99594800/111903891      89%&lt;br/&gt;
                99596500/111903891      89%&lt;br/&gt;
                99597500/111903891      89%&lt;br/&gt;
                99598900/111903891      89%&lt;br/&gt;
                99599400/111903891      89%&lt;br/&gt;
                99600200/111903891      89%&lt;br/&gt;
                99600600/111903891      89%&lt;br/&gt;
                99601600/111903891      89%&lt;br/&gt;
                99602600/111903891      89%&lt;br/&gt;
                99603600/111903891      89%&lt;br/&gt;
                99605300/111903891      89%&lt;br/&gt;
                99605800/111903891      89%&lt;br/&gt;
                99606500/111903891      89%&lt;br/&gt;
                99607300/111903891      89%&lt;br/&gt;
                99609500/111903891      89%&lt;br/&gt;
                99610100/111903891      89%&lt;br/&gt;
                99611000/111903891      89%&lt;/p&gt;

&lt;p&gt;Checking the /var/log/messages file, we saw that the linux oom_killer was being invoked and killing mongod.&lt;/p&gt;

&lt;p&gt;when mongo adds to a new member of a replica set, the data seems to transfer over just fine but then it runs out of memory when it is building its indexes &amp;amp; sort tables etc...&lt;/p&gt;

&lt;p&gt;We have other members of the replica set running with 8GB RAM / 1GB swap drive just fine...&lt;/p&gt;

&lt;p&gt;I&apos;ve only experienced oom_killer being invoked once or twice ever, and it&apos;s only when stuff is REALLY bad on the server, and always always something configured wrong and running out of disk space and ram. &lt;/p&gt;

&lt;p&gt;mongo docs declare that mongo isn&apos;t supposed to do this. &lt;/p&gt;

&lt;p&gt;we aren&apos;t booting up small boxes here... this failure is happening on a dual core machine with 8GB ram and a 1GB swap disk. &lt;/p&gt;


&lt;p&gt;_&lt;em&gt;Mongo Docs&lt;/em&gt;_&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;http://www.mongodb.org/display/DOCS/Production+Notes#ProductionNotes-Swap&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Production+Notes#ProductionNotes-Swap&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&quot;Swap&lt;/p&gt;

&lt;p&gt;It is useful for the linux kernel to have swap space to use in emergencies. Because of the way MongoDB memory maps the database files none of this data will ever end up in swap; this means that on a healthy system the swap space will rarely be used on a system only running MongoDB. Having swap can keep the kernel from killing MongoDB when physical memory limits are reached.&quot;&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;http://www.mongodb.org/display/DOCS/The+Linux+Out+of+Memory+OOM+Killer&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/The+Linux+Out+of+Memory+OOM+Killer&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&quot;You may also want to look at using something which compresses swap/memory like compcache.&lt;/p&gt;

&lt;p&gt;MongoDB uses memory mapped files. The entire data is mapped. Over time, if there is no memory pressure, the mongod resident bytes may approach total memory, as the resident bytes includes file system cache bytes for the file pages open and touched by mongod.&quot;&lt;/p&gt;

&lt;p&gt;_&lt;em&gt;/var/log/messages&lt;/em&gt;_&lt;/p&gt;

&lt;p&gt;and the oom info in the /var/log/messages file:&lt;/p&gt;

&lt;p&gt;Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.177205&amp;#93;&lt;/span&gt; mongod invoked oom-killer: gfp_mask=0x201da, order=0, oom_adj=0, oom_score_adj=0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.178669&amp;#93;&lt;/span&gt; mongod cpuset=/ mems_allowed=0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.179406&amp;#93;&lt;/span&gt; Pid: 11935, comm: mongod Not tainted 3.1.10joyent-centos-6-opt #1&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.180830&amp;#93;&lt;/span&gt; Call Trace:&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.181467&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810a16a4&amp;gt;&amp;#93;&lt;/span&gt; dump_header+0x84/0x1e0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.182273&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810a13aa&amp;gt;&amp;#93;&lt;/span&gt; ? find_lock_task_mm+0x2a/0x70&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.183125&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810a1c5d&amp;gt;&amp;#93;&lt;/span&gt; oom_kill_process+0x7d/0x270&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.183926&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810a2058&amp;gt;&amp;#93;&lt;/span&gt; out_of_memory+0x208/0x3a0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.184715&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810a5eb7&amp;gt;&amp;#93;&lt;/span&gt; __alloc_pages_nodemask+0x787/0x7a0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.185548&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810d54c0&amp;gt;&amp;#93;&lt;/span&gt; alloc_pages_current+0xa0/0x100&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.186367&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8109eabf&amp;gt;&amp;#93;&lt;/span&gt; __page_cache_alloc+0x7f/0x90&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.187178&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8109e589&amp;gt;&amp;#93;&lt;/span&gt; ? find_get_page+0x19/0xa0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.187966&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8109f71b&amp;gt;&amp;#93;&lt;/span&gt; filemap_fault+0x1ab/0x4d0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.188755&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810bd084&amp;gt;&amp;#93;&lt;/span&gt; __do_fault+0x74/0x510&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.189522&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810bd59f&amp;gt;&amp;#93;&lt;/span&gt; handle_pte_fault+0x7f/0xa70&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.190404&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810e184c&amp;gt;&amp;#93;&lt;/span&gt; ? mem_cgroup_count_vm_event+0x1c/0x50&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.191307&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810be0ce&amp;gt;&amp;#93;&lt;/span&gt; handle_mm_fault+0x13e/0x250&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.192142&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff81025c9d&amp;gt;&amp;#93;&lt;/span&gt; do_page_fault+0x12d/0x3b0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.192972&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff81001037&amp;gt;&amp;#93;&lt;/span&gt; ? __switch_to+0x227/0x2f0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.193806&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8157ad85&amp;gt;&amp;#93;&lt;/span&gt; ? __schedule+0x375/0x7c0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.194613&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8157d62f&amp;gt;&amp;#93;&lt;/span&gt; page_fault+0x1f/0x30&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.195403&amp;#93;&lt;/span&gt; Mem-Info:&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.196022&amp;#93;&lt;/span&gt; Node 0 DMA per-cpu:&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.196720&amp;#93;&lt;/span&gt; CPU    0: hi:    0, btch:   1 usd:   0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.197481&amp;#93;&lt;/span&gt; CPU    1: hi:    0, btch:   1 usd:   0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.198255&amp;#93;&lt;/span&gt; Node 0 DMA32 per-cpu:&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.198962&amp;#93;&lt;/span&gt; CPU    0: hi:  186, btch:  31 usd:  82&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.199727&amp;#93;&lt;/span&gt; CPU    1: hi:  186, btch:  31 usd: 153&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.200558&amp;#93;&lt;/span&gt; Node 0 Normal per-cpu:&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.201260&amp;#93;&lt;/span&gt; CPU    0: hi:  186, btch:  31 usd: 175&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.202059&amp;#93;&lt;/span&gt; CPU    1: hi:  186, btch:  31 usd:  32&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.202891&amp;#93;&lt;/span&gt; active_anon:1665611 inactive_anon:301926 isolated_anon:0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.202893&amp;#93;&lt;/span&gt;  active_&lt;a href=&quot;file:11&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:11&lt;/a&gt; inactive_&lt;a href=&quot;file:1308&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:1308&lt;/a&gt; isolated_&lt;a href=&quot;file:31&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:31&lt;/a&gt;&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.202894&amp;#93;&lt;/span&gt;  unevictable:0 dirty:0 writeback:0 unstable:0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.202896&amp;#93;&lt;/span&gt;  free:11493 slab_reclaimable:679 slab_unreclaimable:1676&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.202897&amp;#93;&lt;/span&gt;  mapped:98 shmem:1 pagetables:37659 bounce:0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.207035&amp;#93;&lt;/span&gt; Node 0 DMA free:15908kB min:20kB low:24kB high:28kB active_anon:0kB inactive_anon:0kB active_&lt;a href=&quot;file:0kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0kB&lt;/a&gt; inactive_&lt;a href=&quot;file:0kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0kB&lt;/a&gt; unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15652kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.212201&amp;#93;&lt;/span&gt; lowmem_reserve[]: 0 3504 8040 8040&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.213231&amp;#93;&lt;/span&gt; Node 0 DMA32 free:23140kB min:4996kB low:6244kB high:7492kB active_anon:2913228kB inactive_anon:582760kB active_&lt;a href=&quot;file:72kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:72kB&lt;/a&gt; inactive_&lt;a href=&quot;file:0kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0kB&lt;/a&gt; unevictable:0kB isolated(anon):0kB isolated(file):0kB present:3588340kB mlocked:0kB dirty:0kB writeback:0kB mapped:4kB shmem:4kB slab_reclaimable:364kB slab_unreclaimable:336kB kernel_stack:16kB pagetables:57444kB unstable:0kB bounce:0kB writeback_tmp:0kB pages_scanned:2864 all_unreclaimable? yes&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.218476&amp;#93;&lt;/span&gt; lowmem_reserve[]: 0 0 4536 4536&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.219331&amp;#93;&lt;/span&gt; Node 0 Normal free:6924kB min:6468kB low:8084kB high:9700kB active_anon:3749216kB inactive_anon:624944kB active_&lt;a href=&quot;file:0kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:0kB&lt;/a&gt; inactive_&lt;a href=&quot;file:5104kB&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:5104kB&lt;/a&gt; unevictable:0kB isolated(anon):0kB isolated(file):124kB present:4644864kB mlocked:0kB dirty:0kB writeback:0kB mapped:560kB shmem:0kB slab_reclaimable:2352kB slab_unreclaimable:6368kB kernel_stack:800kB pagetables:93192kB unstable:0kB bounce:0kB writeback_tmp:0kB pages_scanned:157 all_unreclaimable? no&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.224709&amp;#93;&lt;/span&gt; lowmem_reserve[]: 0 0 0 0&lt;br/&gt;
Jun 20 08:05:36 c4fabbdf-f958-4e1f-b159-c6ff18df2020 kernel: &lt;span class=&quot;error&quot;&gt;&amp;#91;286991.225523&amp;#93;&lt;/span&gt; Node 0 DMA: 1*4kB 0*8kB 0*16kB 1*32kB 2*64kB 1*128kB 1*256kB 0*512kB 1*1024kB 1*2048kB 3*4096kB = 15908kB&lt;/p&gt;</comment>
                            <comment id="92226" author="dan@10gen.com" created="Sat, 25 Feb 2012 05:31:01 +0000"  >&lt;p&gt;Glad to hear you&apos;re back up.&lt;/p&gt;</comment>
                            <comment id="90422" author="lujeni" created="Tue, 21 Feb 2012 14:54:34 +0000"  >&lt;p&gt;yes, the sync was very long but without problems.&lt;br/&gt;
However, we encountered a problem unrelated to mongo which forcing to stop mongo.&lt;br/&gt;
We decide to sync data &quot;manual&apos;s way&quot; and it seems to work. (ticket solved)&lt;/p&gt;

&lt;p&gt;Sorry for my bad english and thanks for your troubleshooting.&lt;/p&gt;</comment>
                            <comment id="87170" author="dan@10gen.com" created="Fri, 10 Feb 2012 04:53:22 +0000"  >&lt;p&gt;Hi Julien,&lt;br/&gt;
I don&apos;t exactly understand what you mean.  Do you mean a full sync of the secondary node has been in progress for two weeks, but that the sync has passed the point of building the second index? What do you get when you run db.printReplicationInfo() on the secondary instance?&lt;/p&gt;</comment>
                            <comment id="86924" author="lujeni" created="Thu, 9 Feb 2012 16:35:45 +0000"  >&lt;p&gt;Hello,&lt;/p&gt;

&lt;p&gt;rsync is in progress since 2weeks ago !&lt;br/&gt;
However, no crash and i see in log that XXX objects were cloned.&lt;/p&gt;</comment>
                            <comment id="83648" author="dan@10gen.com" created="Tue, 31 Jan 2012 14:16:37 +0000"  >&lt;p&gt;Hi Julien, did the latest sync attempt work out?&lt;/p&gt;</comment>
                            <comment id="81912" author="lujeni" created="Tue, 24 Jan 2012 09:34:38 +0000"  >&lt;p&gt;Hello,&lt;/p&gt;

&lt;p&gt;I Upgrade my memory server on 13go to 32go. &lt;br/&gt;
Rsync seems good (in duration). Sorry for waste time, if it&apos;s was only memory issue.&lt;br/&gt;
I will upgrade my ticket if full rsync are goood.&lt;/p&gt;</comment>
                            <comment id="79908" author="dan@10gen.com" created="Sat, 14 Jan 2012 22:09:47 +0000"  >&lt;p&gt;Sorry for the delay in response Julien.  Can you attach the full compressed log file from the secondary instance if this problem is still happening?  Can you also post the collection stats? &lt;/p&gt;</comment>
                            <comment id="77442" author="lujeni" created="Wed, 4 Jan 2012 08:50:21 +0000"  >&lt;p&gt;nothing special in /var/log.&lt;/p&gt;

&lt;p&gt;13go RAM. It seems enough to store all index.&lt;br/&gt;
On my primary, db.stats() -&amp;gt; indexSize : 8192&lt;/p&gt;

&lt;p&gt;edit: primay have 64go ram.&lt;/p&gt;</comment>
                            <comment id="77429" author="eliot" created="Wed, 4 Jan 2012 06:52:20 +0000"  >&lt;p&gt;Can you check for something in /var/log/messages?&lt;br/&gt;
How much ram do you have?&lt;/p&gt;</comment>
                            <comment id="77240" author="lujeni" created="Tue, 3 Jan 2012 17:04:27 +0000"  >&lt;p&gt;Last log&apos;s entry:&lt;/p&gt;

&lt;p&gt;....&lt;br/&gt;
180318900/197196147     91%&lt;br/&gt;
181133200/197196147     91%&lt;br/&gt;
181927300/197196147     92%&lt;br/&gt;
182778000/197196147     92%&lt;/p&gt;

&lt;p&gt;Nothing else&lt;/p&gt;
</comment>
                            <comment id="77221" author="scotthernandez" created="Tue, 3 Jan 2012 16:24:58 +0000"  >&lt;p&gt;What is in the logs when it crashes?&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>12.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 3 Jan 2012 16:24:58 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        11 years, 35 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ian@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            11 years, 35 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>eliot</customfieldvalue>
            <customfieldvalue>cb_john</customfieldvalue>
            <customfieldvalue>lujeni</customfieldvalue>
            <customfieldvalue>scotthernandez</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrohxr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrithz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>23432</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10166" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Tests Written</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10153"><![CDATA[Unneeded]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht0al3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>