<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:47:56 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-18519] Using $out in aggregation appears to leak non-mapped virtual memory</title>
                <link>https://jira.mongodb.org/browse/SERVER-18519</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Using the c# driver, we run 200K+ aggregation jobs on a nightly basis where each uses $out to create a temporary collection. These collections are short lived and dropped after their data is moved to a permanent collection. Every time our jobs run we notice an increase of about 0.5gb of non-mapped virtual memory, which never releases, and in turn requires a periodic node re-start to free-up memory.&lt;/p&gt;</description>
                <environment></environment>
        <key id="204888">SERVER-18519</key>
            <summary>Using $out in aggregation appears to leak non-mapped virtual memory</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="2">Won&apos;t Fix</resolution>
                                        <assignee username="backlog-server-query">Backlog - Query Team</assignee>
                                    <reporter username="SigNarvaez">Sigfrido Narvaez [X]</reporter>
                        <labels>
                    </labels>
                <created>Mon, 18 May 2015 17:15:40 +0000</created>
                <updated>Tue, 6 Dec 2022 04:51:16 +0000</updated>
                            <resolved>Tue, 23 Feb 2016 16:49:34 +0000</resolved>
                                    <version>2.6.9</version>
                                                    <component>Aggregation Framework</component>
                                        <votes>0</votes>
                                    <watches>12</watches>
                                                                                                                <comments>
                            <comment id="1181995" author="charlie.swanson" created="Tue, 23 Feb 2016 16:49:34 +0000"  >&lt;p&gt;I&apos;m closing this ticket as &quot;Won&apos;t Fix&quot;. The problem doesn&apos;t appear to affect 3.0, 3.2, or master, and we will not be doing a targeted patch for the 2.6 branch. If this problem is observed on any version since 3.0, please feel free to re-open.&lt;/p&gt;</comment>
                            <comment id="966957" author="signarvaez" created="Wed, 15 Jul 2015 17:53:56 +0000"  >&lt;p&gt;Update: We are still using 2.6 and have released a change on our code base to only use $out when needed (if an aggregation exceeds the 16mb limit). This has stopped the memory leak. We will upgrade to 3.0.4 soon and will report back.&lt;/p&gt;

&lt;p&gt;Cheers,&lt;/p&gt;

&lt;p&gt;Sig&lt;/p&gt;</comment>
                            <comment id="943253" author="ramon.fernandez" created="Wed, 17 Jun 2015 18:00:56 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=SigNarvaez&quot; class=&quot;user-hover&quot; rel=&quot;SigNarvaez&quot;&gt;SigNarvaez&lt;/a&gt;, further investigation shows that only 2.6 is affected by increased memory consumption when using $out. In 3.0 there&apos;s an apparent increase, but this increase it&apos;s for data structures used by the lock manager that are later released and collected for reuse every minute.&lt;/p&gt;

&lt;p&gt;Would it be possible for you to test your application against a 3.0.4 node and report back? Would be useful to confirm that 3.0 addressed fixes your use case.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="931220" author="signarvaez" created="Thu, 4 Jun 2015 03:13:37 +0000"  >&lt;p&gt;Excellent! Thanks for looking into this.&lt;/p&gt;</comment>
                            <comment id="930868" author="ramon.fernandez" created="Wed, 3 Jun 2015 19:13:38 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=SigNarvaez&quot; class=&quot;user-hover&quot; rel=&quot;SigNarvaez&quot;&gt;SigNarvaez&lt;/a&gt;, nice meeting you today. This is to let you know that we&apos;ve been able to reproduce the memory increase you observe and we&apos;re investigating. Credit goes to &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=craiggwilson&quot; class=&quot;user-hover&quot; rel=&quot;craiggwilson&quot;&gt;craiggwilson&lt;/a&gt; for finding a fast repro!&lt;/p&gt;</comment>
                            <comment id="925119" author="signarvaez" created="Wed, 27 May 2015 19:40:34 +0000"  >&lt;p&gt;Stats taken during peak time of aggregation jobs&lt;/p&gt;</comment>
                            <comment id="925117" author="signarvaez" created="Wed, 27 May 2015 19:39:45 +0000"  >&lt;p&gt;Aggregations&lt;br/&gt;
var maxTime = new TimeSpan(0, 0, 30, 0); // 30 minute time allowed for aggregations &lt;br/&gt;
var aggregateArgs = new AggregateArgs &lt;/p&gt;
{ 
	AllowDiskUse = true, 
	OutputMode = AggregateOutputMode.Inline, 
	MaxTime = maxTime, 
	Pipeline = pipe
}
&lt;p&gt;;&lt;/p&gt;

&lt;p&gt;All queries use this connection profile:&lt;br/&gt;
var clientSettings = new MongoClientSettings {&lt;br/&gt;
	Credentials = credentials,&lt;br/&gt;
	Servers = mongoUrl.Servers,&lt;br/&gt;
	ConnectTimeout = TimeSpan.FromMinutes(1),&lt;br/&gt;
	MaxConnectionIdleTime = TimeSpan.FromMinutes(1),&lt;br/&gt;
	WriteConcern = new WriteConcern &lt;/p&gt;
{ W = 1 }
&lt;p&gt;, }; &lt;/p&gt;

&lt;p&gt;return new MongoClient(clientSettings);&lt;/p&gt;

&lt;p&gt;server stats attached. DB, collection names and IP&apos;s are masked.&lt;/p&gt;</comment>
                            <comment id="917961" author="samk" created="Tue, 19 May 2015 14:59:50 +0000"  >&lt;p&gt;Hello, &lt;/p&gt;

&lt;p&gt;In cases of apparent memory leaks, we want to be sure to rule out a case where open cursors or connections are the ones consuming resources. You can see current operations using &lt;tt&gt;db.currentOp();&lt;/tt&gt; and the output of &lt;tt&gt;db.serverStatus().metrics.cursor&lt;/tt&gt; in the &lt;tt&gt;mongo&lt;/tt&gt; shell.  Can you please post the output of those operations you can post them to this ticket so that we can look over them together?&lt;/p&gt;

&lt;p&gt;Additionally, can you check your application to see you&apos;re setting the &lt;tt&gt;noTimeout&lt;/tt&gt; option, or setting a custom timeout for your queries in your code? This can allow us to detect the source of non-expiring cursors that are not otherwise accounted for. &lt;/p&gt;

&lt;p&gt;Cheers,&lt;br/&gt;
sam&lt;/p&gt;</comment>
                            <comment id="916934" author="signarvaez" created="Mon, 18 May 2015 17:21:38 +0000"  >&lt;p&gt;The image &quot;memory increase 2 - fix.jpg&quot; shows that the increase stopped after we moved away from using $out. Other aggregations that didn&apos;t require $out did not cause an increase, this can be seen in the other images.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="75932" name="Mongo_stats_masked.txt" size="28982" author="SigNarvaez" created="Wed, 27 May 2015 19:40:34 +0000"/>
                            <attachment id="74738" name="memory increase 1.jpg" size="239419" author="SigNarvaez" created="Mon, 18 May 2015 17:26:22 +0000"/>
                            <attachment id="74739" name="memory increase 2 - Fix.jpg" size="219739" author="SigNarvaez" created="Mon, 18 May 2015 17:26:22 +0000"/>
                            <attachment id="74740" name="non-mapped virtual memory.png" size="67633" author="SigNarvaez" created="Mon, 18 May 2015 17:26:22 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25143"><![CDATA[Query]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 18 May 2015 17:32:00 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 51 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>alexander.golin@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 51 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>backlog-server-query</customfieldvalue>
            <customfieldvalue>charlie.swanson@mongodb.com</customfieldvalue>
            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>
            <customfieldvalue>sam.kleinman</customfieldvalue>
            <customfieldvalue>SigNarvaez</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrl5q7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hshi7r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;&lt;a href=&quot;https://mms.mongodb.com/host/chartReplSetHosts/55079b4ce4b062b9d58cc288/55079be7e4b0dbee2a245c72#chartLinkHourOneMonth&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://mms.mongodb.com/host/chartReplSetHosts/55079b4ce4b062b9d58cc288/55079be7e4b0dbee2a245c72#chartLinkHourOneMonth&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://mms.mongodb.com/host/chartReplSetHosts/541a0803e4b07a2f96a3518a/541a0cabe4b0539fc6ff68d4&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://mms.mongodb.com/host/chartReplSetHosts/541a0803e4b07a2f96a3518a/541a0cabe4b0539fc6ff68d4&lt;/a&gt;&lt;/p&gt;
</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrj967:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>