<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 08:34:21 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[GODRIVER-507] bson.(*decoder).Decode performance is very poor, how to improve it </title>
                <link>https://jira.mongodb.org/browse/GODRIVER-507</link>
                <project id="14289" key="GODRIVER">Go Driver</project>
                    <description>&lt;p&gt;when i use mongodb-go-driver load 300,000 records , the total size is about 130MB, i use runtime.pprof to analyse performance of cpu and memory, i found that cur.Decode(elem) use lots of cpu and memory. can anyone tell me how to improve it&lt;/p&gt;


&lt;p&gt;cpu analysis&lt;/p&gt;

&lt;p&gt;macbookpro:fpr_index fredlee$ go tool pprof fpr_index  logs_profile/cpu-profile-2018-07-21_18-38-28.prof &lt;br/&gt;
File: fpr_index&lt;br/&gt;
Type: cpu&lt;br/&gt;
Time: Jul 21, 2018 at 6:38pm (CST)&lt;br/&gt;
Duration: 2.90mins, Total samples = 2.45mins (84.23%)&lt;br/&gt;
Entering interactive mode (type &quot;help&quot; for commands, &quot;o&quot; for options)&lt;br/&gt;
(pprof) top20 -cum&lt;br/&gt;
Showing nodes accounting for 43.10s, 29.37% of 146.74s total&lt;br/&gt;
Dropped 453 nodes (cum &amp;lt;= 0.73s)&lt;br/&gt;
Showing top 20 nodes out of 119&lt;br/&gt;
      flat  flat%   sum%        cum   cum%&lt;br/&gt;
         0     0%     0%     97.01s 66.11%  huoli/fpr_index/singleton.(*M1).handle&lt;br/&gt;
     0.05s 0.034% 0.034%     90.02s 61.35%  huoli/fpr_index/util.(*Orderinfos).LoadFromDb&lt;br/&gt;
     0.05s 0.034% 0.068%     82.30s 56.09%  github.com/mongodb/mongo-go-driver/core/topology.(*cursor).Decode&lt;br/&gt;
     0.10s 0.068%  0.14%     82.01s 55.89%  github.com/mongodb/mongo-go-driver/bson.(*decoder).Decode&lt;br/&gt;
     0.03s  0.02%  0.16%     81.93s 55.83%  github.com/mongodb/mongo-go-driver/bson.(*decoder).reflectDecode&lt;br/&gt;
     0.32s  0.22%  0.37%     81.83s 55.77%  github.com/mongodb/mongo-go-driver/bson.(*decoder).decodeIntoStruct&lt;br/&gt;
         0     0%  0.37%     80.62s 54.94%  huoli/fpr_index/singleton.(*M1).check.func1&lt;br/&gt;
     0.14s 0.095%  0.47%     67.86s 46.25%  reflect.Value.FieldByNameFunc&lt;br/&gt;
     0.15s   0.1%  0.57%     67.32s 45.88%  reflect.(*rtype).FieldByNameFunc&lt;br/&gt;
     2.55s  1.74%  2.31%     66.87s 45.57%  reflect.(*structType).FieldByNameFunc&lt;br/&gt;
     0.73s   0.5%  2.81%     62.37s 42.50%  github.com/mongodb/mongo-go-driver/bson.(*decoder).decodeIntoStruct.func1&lt;br/&gt;
     2.74s  1.87%  4.67%     54.65s 37.24%  github.com/mongodb/mongo-go-driver/bson.matchesField&lt;br/&gt;
     0.10s 0.068%  4.74%     50.52s 34.43%  runtime.systemstack&lt;br/&gt;
         0     0%  4.74%     43.06s 29.34%  runtime.gcBgMarkWorker&lt;br/&gt;
         0     0%  4.74%     43.05s 29.34%  runtime.gcBgMarkWorker.func2&lt;br/&gt;
     0.46s  0.31%  5.06%     43.05s 29.34%  runtime.gcDrain&lt;br/&gt;
    15.71s 10.71% 15.76%     42.92s 29.25%  runtime.scanobject&lt;br/&gt;
     0.83s  0.57% 16.33%     38.62s 26.32%  github.com/mongodb/mongo-go-driver/bson.(*decoder).getReflectValue&lt;br/&gt;
     0.04s 0.027% 16.36%     36.93s 25.17%  github.com/mongodb/mongo-go-driver/bson.(*decoder).decodeBSONArrayToSlice&lt;br/&gt;
    19.10s 13.02% 29.37%     26.71s 18.20%  runtime.mallocgc&lt;br/&gt;
(pprof) web&lt;br/&gt;
(pprof) list LoadFromDb&lt;br/&gt;
Total: 2.45mins&lt;br/&gt;
ROUTINE ======================== huoli/fpr_index/util.(*Orderinfos).LoadFromDb in /Users/fredlee/Documents/develop/go/workspace/src/huoli/fpr_index/util/mongo_orderinfo.go&lt;br/&gt;
      30ms   1.50mins (flat, cum) 61.33% of Total&lt;br/&gt;
         .          .     67:	}&lt;br/&gt;
         .          .     68:	defer mongo.Close()&lt;br/&gt;
         .          .     69:	coll := mongo.C()&lt;br/&gt;
         .          .     70:&lt;br/&gt;
         .          .     71:	_map := make(map&lt;span class=&quot;error&quot;&gt;&amp;#91;string&amp;#93;&lt;/span&gt;Orderinfo)&lt;br/&gt;
         .       10ms     72:	cur, err := coll.Find(context.Background(), nil)&lt;br/&gt;
         .          .     73:	if err != nil &lt;/p&gt;
{
         .          .     74:		return num, err
         .          .     75:	}
&lt;p&gt;         .          .     76:	defer cur.Close(context.Background())&lt;br/&gt;
         .      6.63s     77:	for cur.Next(context.Background()) {&lt;br/&gt;
         .      200ms     78:		elem := Orderinfo{}&lt;br/&gt;
      10ms   1.37mins     79:		err1 := cur.Decode(&amp;amp;elem)&lt;br/&gt;
         .          .     80:		if err1 != nil &lt;/p&gt;
{
         .          .     81:			//_mylog := mylog.GetMylog().Regular()
         .          .     82:			//_mylog.Errorf(&quot;cur.Decode(elem) fail,%v&quot;, err1)
         .          .     83:			log.Printf(&quot;cur.Decode(elem) fail,%v\n&quot;, err1)
         .          .     84:		}
&lt;p&gt; else &lt;/p&gt;
{
      20ms      850ms     85:			_map[elem.Id] = elem
         .          .     86:		}
&lt;p&gt;         .          .     87:	}&lt;br/&gt;
         .          .     88:	num = len(_map)&lt;br/&gt;
         .          .     89:	if num &amp;gt; 0 { //&#34920;&#37324;&#26080;&#25968;&#25454;,&#20173;&#20351;&#29992;&#26087;&#25968;&#25454;&lt;br/&gt;
         .          .     90:		m._map = _map&lt;/p&gt;


&lt;p&gt;memory analysis&lt;br/&gt;
192:fpr_index fredlee$ go tool pprof fpr_index  logs_profile/mem-profile-2018-07-21_18-41-20.prof &lt;br/&gt;
File: fpr_index&lt;br/&gt;
Type: inuse_space&lt;br/&gt;
Time: Jul 21, 2018 at 6:41pm (CST)&lt;br/&gt;
Entering interactive mode (type &quot;help&quot; for commands, &quot;o&quot; for options)&lt;br/&gt;
(pprof) web&lt;br/&gt;
(pprof) top10 -cum&lt;br/&gt;
Showing nodes accounting for 221.24MB, 64.69% of 342MB total&lt;br/&gt;
Dropped 17 nodes (cum &amp;lt;= 1.71MB)&lt;br/&gt;
Showing top 10 nodes out of 37&lt;br/&gt;
      flat  flat%   sum%        cum   cum%&lt;br/&gt;
         0     0%     0%   339.78MB 99.35%  huoli/fpr_index/singleton.(*M1).check.func1&lt;br/&gt;
         0     0%     0%   339.78MB 99.35%  huoli/fpr_index/singleton.(*M1).handle&lt;br/&gt;
  105.73MB 30.92% 30.92%   339.28MB 99.20%  huoli/fpr_index/util.(*Orderinfos).LoadFromDb&lt;br/&gt;
         0     0% 30.92%   221.51MB 64.77%  github.com/mongodb/mongo-go-driver/bson.(*decoder).Decode&lt;br/&gt;
         0     0% 30.92%   221.51MB 64.77%  github.com/mongodb/mongo-go-driver/bson.(*decoder).decodeIntoStruct&lt;br/&gt;
         0     0% 30.92%   221.51MB 64.77%  github.com/mongodb/mongo-go-driver/bson.(*decoder).reflectDecode&lt;br/&gt;
         0     0% 30.92%   221.51MB 64.77%  github.com/mongodb/mongo-go-driver/core/topology.(*cursor).Decode&lt;br/&gt;
         0     0% 30.92%   215.01MB 62.87%  github.com/mongodb/mongo-go-driver/bson.(*decoder).getReflectValue&lt;br/&gt;
         0     0% 30.92%   136.51MB 39.92%  github.com/mongodb/mongo-go-driver/bson.(*decoder).decodeBSONArrayToSlice&lt;br/&gt;
  115.50MB 33.77% 64.69%   115.50MB 33.77%  github.com/mongodb/mongo-go-driver/bson.(*Value).StringValue&lt;br/&gt;
(pprof) list LoadFromDb&lt;br/&gt;
Total: 342MB&lt;br/&gt;
ROUTINE ======================== huoli/fpr_index/util.(*Orderinfos).LoadFromDb in /Users/fredlee/Documents/develop/go/workspace/src/huoli/fpr_index/util/mongo_orderinfo.go&lt;br/&gt;
  105.73MB   339.28MB (flat, cum) 99.20% of Total&lt;br/&gt;
         .          .     72:	cur, err := coll.Find(context.Background(), nil)&lt;br/&gt;
         .          .     73:	if err != nil &lt;/p&gt;
{
         .          .     74:		return num, err
         .          .     75:	}
&lt;p&gt;         .          .     76:	defer cur.Close(context.Background())&lt;br/&gt;
         .    12.03MB     77:	for cur.Next(context.Background()) {&lt;br/&gt;
  512.09kB   512.09kB     78:		elem := Orderinfo{}&lt;br/&gt;
         .   221.51MB     79:		err1 := cur.Decode(&amp;amp;elem)&lt;br/&gt;
         .          .     80:		if err1 != nil &lt;/p&gt;
{
         .          .     81:			//_mylog := mylog.GetMylog().Regular()
         .          .     82:			//_mylog.Errorf(&quot;cur.Decode(elem) fail,%v&quot;, err1)
         .          .     83:			log.Printf(&quot;cur.Decode(elem) fail,%v\n&quot;, err1)
         .          .     84:		}
&lt;p&gt; else &lt;/p&gt;
{
  105.23MB   105.23MB     85:			_map[elem.Id] = elem
         .          .     86:		}
&lt;p&gt;         .          .     87:	}&lt;br/&gt;
         .          .     88:	num = len(_map)&lt;br/&gt;
         .          .     89:	if num &amp;gt; 0 { //&#34920;&#37324;&#26080;&#25968;&#25454;,&#20173;&#20351;&#29992;&#26087;&#25968;&#25454;&lt;br/&gt;
         .          .     90:		m._map = _map&lt;/p&gt;
</description>
                <environment>go1.10.3, linux</environment>
        <key id="574506">GODRIVER-507</key>
            <summary>bson.(*decoder).Decode performance is very poor, how to improve it </summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13203">Gone away</resolution>
                                        <assignee username="-1">Unassigned</assignee>
                                    <reporter username="jackylee">orange.jackylee</reporter>
                        <labels>
                    </labels>
                <created>Sat, 21 Jul 2018 11:54:59 +0000</created>
                <updated>Fri, 27 Oct 2023 20:01:34 +0000</updated>
                            <resolved>Thu, 10 Jan 2019 20:48:51 +0000</resolved>
                                    <version>0.0.9</version>
                                                    <component>BSON</component>
                    <component>Performance</component>
                                        <votes>0</votes>
                                    <watches>1</watches>
                                                                                                                <comments>
                            <comment id="2111612" author="ian@10gen.com" created="Thu, 10 Jan 2019 20:48:51 +0000"  >&lt;p&gt;closing as gone away since we have since rewritten the bson decoder.&#160; please feel free to open a new ticket with any new performance numbers you might have.&lt;/p&gt;</comment>
                            <comment id="1963639" author="jackylee" created="Thu, 2 Aug 2018 03:06:44 +0000"  >&lt;p&gt;thank you  Kris Brandow&lt;br/&gt;
hope more efficiency decoder&lt;/p&gt;</comment>
                            <comment id="1960705" author="kris.brandow" created="Mon, 30 Jul 2018 20:04:42 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jackylee&quot; class=&quot;user-hover&quot; rel=&quot;jackylee&quot;&gt;jackylee&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thanks for the report! We&apos;ve noticed this in other areas as well. We are currently working on a new implementation to solve performance issues with the BSON decoder. You can track the progress of this in &lt;a href=&quot;https://jira.mongodb.org/browse/GODRIVER-494&quot; title=&quot;BSON Codec Redesign&quot; class=&quot;issue-link&quot; data-issue-key=&quot;GODRIVER-494&quot;&gt;&lt;del&gt;GODRIVER-494&lt;/del&gt;&lt;/a&gt;, which has been linked to this ticket.&lt;/p&gt;

&lt;p&gt;Thanks!&lt;/p&gt;

&lt;p&gt;--Kris&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="572145">GODRIVER-494</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr8c3z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            </customfields>
    </item>
</channel>
</rss>