Uploaded image for project: 'Go Driver'
  1. Go Driver
  2. GODRIVER-1138

size of collection is 3.6MB, load from mongo with v1.0.3, profile it use 26MB

    XMLWordPrintable

    Details

    • Type: Improvement
    • Status: Closed
    • Priority: Major - P3
    • Resolution: Gone away
    • Affects Version/s: 1.0.3
    • Fix Version/s: None
    • Component/s: CRUD
    • Labels:
      None
    • Environment:
      go version go1.12.5
      Linux VM-0-4-ubuntu 4.4.0-130-generic #156-Ubuntu SMP Thu Jun 14 08:53:28 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

      Description

      size of collection is 3.6MB

      {
      "ns" : "test.globalAirport",
      "size" : 3735316,
      "count" : 5320,
      "avgObjSize" : 702,
      "storageSize" : 1646592,
      "capped" : false,
      "wiredTiger" : {
      "metadata" :

      { "formatVersion" : 1 }

      ,
      "creationString" : "access_pattern_hint=none,allocation_size=4KB,app_metadata=(formatVersion=1),assert=(commit_timestamp=none,read_timestamp=none),block_allocation=best,block_compressor=snappy,cache_resident=false,checksum=on,colgroups=,collator=,columns=,dictionary=0,encryption=(keyid=,name=),exclusive=false,extractor=,format=btree,huffman_key=,huffman_value=,ignore_in_memory_cache_size=false,immutable=false,internal_item_max=0,internal_key_max=0,internal_key_truncate=true,internal_page_max=4KB,key_format=q,key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=64MB,log=(enabled=true),lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8,bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB,merge_custom=(prefix=,start_generation=0,suffix=),merge_max=15,merge_min=0),memory_page_image_max=0,memory_page_max=10m,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false,prefix_compression_min=4,source=,split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90,type=file,value_format=u",
      "type" : "file",
      "uri" : "statistics:table:collection-0-3820145496175747809",
      "LSM" :

      { "bloom filter false positives" : 0, "bloom filter hits" : 0, "bloom filter misses" : 0, "bloom filter pages evicted from cache" : 0, "bloom filter pages read into cache" : 0, "bloom filters in the LSM tree" : 0, "chunks in the LSM tree" : 0, "highest merge generation in the LSM tree" : 0, "queries that could have benefited from a Bloom filter that did not exist" : 0, "sleep for LSM checkpoint throttle" : 0, "sleep for LSM merge throttle" : 0, "total size of bloom filters" : 0 }

      ,
      "block-manager" :

      { "allocations requiring file extension" : 0, "blocks allocated" : 0, "blocks freed" : 0, "checkpoint size" : 1626112, "file allocation unit size" : 4096, "file bytes available for reuse" : 4096, "file magic number" : 120897, "file major version number" : 1, "file size in bytes" : 1646592, "minor version number" : 0 }

      ,
      "btree" :

      { "btree checkpoint generation" : 327, "column-store fixed-size leaf pages" : 0, "column-store internal pages" : 0, "column-store variable-size RLE encoded values" : 0, "column-store variable-size deleted values" : 0, "column-store variable-size leaf pages" : 0, "fixed-record size" : 0, "maximum internal page key size" : 368, "maximum internal page size" : 4096, "maximum leaf page key size" : 2867, "maximum leaf page size" : 32768, "maximum leaf page value size" : 67108864, "maximum tree depth" : 3, "number of key/value pairs" : 0, "overflow pages" : 0, "pages rewritten by compaction" : 0, "row-store internal pages" : 0, "row-store leaf pages" : 0 }

      ,
      "cache" :

      { "bytes currently in the cache" : 4143142, "bytes read into cache" : 3774291, "bytes written from cache" : 0, "checkpoint blocked page eviction" : 0, "data source pages selected for eviction unable to be evicted" : 0, "eviction walk passes of a file" : 0, "eviction walk target pages histogram - 0-9" : 0, "eviction walk target pages histogram - 10-31" : 0, "eviction walk target pages histogram - 128 and higher" : 0, "eviction walk target pages histogram - 32-63" : 0, "eviction walk target pages histogram - 64-128" : 0, "eviction walks abandoned" : 0, "eviction walks gave up because they restarted their walk twice" : 0, "eviction walks gave up because they saw too many pages and found no candidates" : 0, "eviction walks gave up because they saw too many pages and found too few candidates" : 0, "eviction walks reached end of tree" : 0, "eviction walks started from root of tree" : 0, "eviction walks started from saved location in tree" : 0, "hazard pointer blocked page eviction" : 0, "in-memory page passed criteria to be split" : 0, "in-memory page splits" : 0, "internal pages evicted" : 0, "internal pages split during eviction" : 0, "leaf pages split during eviction" : 0, "modified pages evicted" : 0, "overflow pages read into cache" : 0, "page split during eviction deepened the tree" : 0, "page written requiring lookaside records" : 0, "pages read into cache" : 135, "pages read into cache after truncate" : 0, "pages read into cache after truncate in prepare state" : 0, "pages read into cache requiring lookaside entries" : 0, "pages requested from the cache" : 2807, "pages seen by eviction walk" : 0, "pages written from cache" : 0, "pages written requiring in-memory restoration" : 0, "tracked dirty bytes in the cache" : 0, "unmodified pages evicted" : 0 }

      ,
      "cache_walk" :

      { "Average difference between current eviction generation when the page was last considered" : 0, "Average on-disk page image size seen" : 0, "Average time in cache for pages that have been visited by the eviction server" : 0, "Average time in cache for pages that have not been visited by the eviction server" : 0, "Clean pages currently in cache" : 0, "Current eviction generation" : 0, "Dirty pages currently in cache" : 0, "Entries in the root page" : 0, "Internal pages currently in cache" : 0, "Leaf pages currently in cache" : 0, "Maximum difference between current eviction generation when the page was last considered" : 0, "Maximum page size seen" : 0, "Minimum on-disk page image size seen" : 0, "Number of pages never visited by eviction server" : 0, "On-disk page image sizes smaller than a single allocation unit" : 0, "Pages created in memory and never written" : 0, "Pages currently queued for eviction" : 0, "Pages that could not be queued for eviction" : 0, "Refs skipped during cache traversal" : 0, "Size of the root page" : 0, "Total number of pages currently in cache" : 0 }

      ,
      "compression" :

      { "compressed pages read" : 134, "compressed pages written" : 0, "page written failed to compress" : 0, "page written was too small to compress" : 0, "raw compression call failed, additional data available" : 0, "raw compression call failed, no additional data available" : 0, "raw compression call succeeded" : 0 }

      ,
      "cursor" :

      { "bulk-loaded cursor-insert calls" : 0, "create calls" : 2, "cursor operation restarted" : 0, "cursor-insert key and value bytes inserted" : 0, "cursor-remove key bytes removed" : 0, "cursor-update value bytes updated" : 0, "cursors cached on close" : 0, "cursors reused from cache" : 33, "insert calls" : 0, "modify calls" : 0, "next calls" : 85338, "prev calls" : 1, "remove calls" : 0, "reserve calls" : 0, "reset calls" : 728, "search calls" : 0, "search near calls" : 656, "truncate calls" : 0, "update calls" : 0 }

      ,
      "reconciliation" :

      { "dictionary matches" : 0, "fast-path pages deleted" : 0, "internal page key bytes discarded using suffix compression" : 0, "internal page multi-block writes" : 0, "internal-page overflow keys" : 0, "leaf page key bytes discarded using prefix compression" : 0, "leaf page multi-block writes" : 0, "leaf-page overflow keys" : 0, "maximum blocks required for a page" : 0, "overflow values written" : 0, "page checksum matches" : 0, "page reconciliation calls" : 0, "page reconciliation calls for eviction" : 0, "pages deleted" : 0 }

      ,
      "session" :

      { "cached cursor count" : 2, "object compaction" : 0, "open cursor count" : 0 }

      ,
      "transaction" :

      { "update conflicts" : 0 }

      },
      "nindexes" : 1,
      "totalIndexSize" : 86016,
      "indexSizes" :

      { "_id_" : 86016 }

      ,
      "ok" : 1.0
      }

       

      mycode

      ..............

      collection := client.Database(mgoc.Database).Collection(GetGlobalAirportCollection().Collection) collection := client.Database(mgoc.Database).Collection(GetGlobalAirportCollection().Collection) ctx, cancle := context.WithTimeout(context.Background(), 30*time.Second) defer cancle() cur, err := collection.Find(ctx, bson.D{}) if err != nil { return } defer cur.Close(ctx) for cur.Next(ctx) { var result GlobAirport err := cur.Decode(&result) if err != nil

      { continue }

      // do something with result.... sli = append(sli, result) } if err = cur.Err(); err != nil { return }

      ..............

       

       

      go profile

      macbookpro:Downloads fredlee$ go tool pprof http://ip:port/debug/pprof/heap

      Fetching profile over HTTP from http://ip:port/debug/pprof/heap

      Saved profile in /Users/fredlee/pprof/pprof.catweb_server.alloc_objects.alloc_space.inuse_objects.inuse_space.009.pb.gz

      File: catweb_server

      Type: inuse_space

      Time: Jun 14, 2019 at 8:55pm (CST)

      Entering interactive mode (type "help" for commands, "o" for options)

      (pprof) top10 -cum

      Showing nodes accounting for 0, 0% of 24.44MB total

      Showing top 10 nodes out of 40

            flat  flat%   sum%        cum   cum%

               0     0%     0%    24.44MB   100%  catweb_server/internal/async.(*M1).Start

               0     0%     0%    24.44MB   100%  catweb_server/internal/async.(*M1).handle

               0     0%     0%    24.44MB   100%  catweb_server/internal/db.(*MongoOfficalClient).Do

               0     0%     0%    24.44MB   100%  catweb_server/util.(*GlobAirports).LoadFromDb

               0     0%     0%    24.44MB   100%  catweb_server/util.(*GlobAirports).LoadFromDb.func1

               0     0%     0%    24.44MB   100%  go.mongodb.org/mongo-driver/mongo.(*Cursor).Next

               0     0%     0%    24.44MB   100%  go.mongodb.org/mongo-driver/x/mongo/driver.(*BatchCursor).Next

               0     0%     0%    24.44MB   100%  go.mongodb.org/mongo-driver/x/mongo/driver.(*BatchCursor).getMore

               0     0%     0%    24.44MB   100%  go.mongodb.org/mongo-driver/x/network/command.(*GetMore).RoundTrip

               0     0%     0%    24.44MB   100%  go.mongodb.org/mongo-driver/x/network/command.(*Read).RoundTrip

      (pprof) list LoadFromDb

      Total: 24.44MB

      ROUTINE ======================== catweb_server/util.(*GlobAirports).LoadFromDb in /Users/fredlee/Documents/project/商城/catweb_server/util/mongo_globalairport.go

               0    24.44MB (flat, cum)   100% of Total

               .          .     71: if err = cur.Err(); err != nil

      {          .          .     72: return          .          .     73: }

               .          .     74: return

               .          .     75: }

               .    24.44MB     76: err = mgoc.Do(handler)

               .          .     77: if err != nil

      {          .          .     78: return          .          .     79: }

               .          .     80:

               .          .     81: num = len(sli)

      ROUTINE ======================== catweb_server/util.(*GlobAirports).LoadFromDb.func1 in /Users/fredlee/Documents/project/商城/catweb_server/util/mongo_globalairport.go

               0    24.44MB (flat, cum)   100% of Total

               .          .     57: cur, err := collection.Find(ctx, bson.D{})

               .          .     58: if err != nil

      {          .          .     59: return          .          .     60: }

               .          .     61: defer cur.Close(ctx)

               .    24.44MB     62: for cur.Next(ctx) {

               .          .     63: var result GlobAirport

               .          .     64: err := cur.Decode(&result)

               .          .     65: if err != nil

      {          .          .     66: continue          .          .     67: }

      (pprof) list Next

      Total: 24.44MB

      ROUTINE ======================== go.mongodb.org/mongo-driver/mongo.(*Cursor).Next in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/mongo/cursor.go

               0    24.44MB (flat, cum)   100% of Total

               .          .     88:

               .          .     89: // call the Next method in a loop until at least one document is returned in the next batch or

               .          .     90: // the context times out.

               .          .     91: for {

               .          .     92: // If we don't have a next batch

               .    24.44MB     93: if !c.bc.Next(ctx) {

               .          .     94: // Do we have an error? If so we return false.

               .          .     95: c.err = c.bc.Err()

               .          .     96: if c.err != nil

      {          .          .     97: return false          .          .     98: }

      ROUTINE ======================== go.mongodb.org/mongo-driver/x/mongo/driver.(*BatchCursor).Next in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/mongo/driver/batch_cursor.go

               0    24.44MB (flat, cum)   100% of Total

               .          .    155: }

               .          .    156:

               .          .    157: if bc.legacy()

      {          .          .    158: bc.legacyGetMore(ctx)          .          .    159: }

      else

      {          .    24.44MB    160: bc.getMore(ctx)          .          .    161: }

               .          .    162:

               .          .    163: switch bc.currentBatch.Style

      {          .          .    164: case bsoncore.SequenceStyle:          .          .    165: return len(bc.currentBatch.Data) > 0 (pprof) list getMore Total: 24.44MB ROUTINE ======================== go.mongodb.org/mongo-driver/x/mongo/driver.(*BatchCursor).getMore in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/mongo/driver/batch_cursor.go          0    24.44MB (flat, cum)   100% of Total          .          .    244: Clock:   bc.clock,          .          .    245: ID:      bc.id,          .          .    246: NS:      bc.namespace,          .          .    247: Opts:    bc.opts,          .          .    248: Session: bc.clientSession,          .    24.44MB    249: }

      ).RoundTrip(ctx, bc.server.SelectedDescription(), conn)

               .          .    250: if err != nil

      {          .          .    251: _ = conn.Close() // The command response error is more important here          .          .    252: bc.err = err          .          .    253: return          .          .    254: }

      (pprof) list RoundTrip

      Total: 24.44MB

      ROUTINE ======================== go.mongodb.org/mongo-driver/x/network/command.(*GetMore).RoundTrip in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/network/command/get_more.go

               0    24.44MB (flat, cum)   100% of Total

               .          .     97: cmd, err := gm.encode(desc)

               .          .     98: if err != nil

      {          .          .     99: return nil, err          .          .    100: }

               .          .    101:

               .    24.44MB    102: rdr, err := cmd.RoundTrip(ctx, desc, rw)

               .          .    103: if err != nil

      {          .          .    104: return nil, err          .          .    105: }

               .          .    106:

               .          .    107: return gm.decode(desc, rdr).Result()

      ROUTINE ======================== go.mongodb.org/mongo-driver/x/network/command.(*Read).RoundTrip in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/network/command/read.go

               0    24.44MB (flat, cum)   100% of Total

               .          .    266: return nil, err

               .          .    267: }

               .          .    268: // Connection errors are transient

               .          .    269: return nil, Error{Message: err.Error(), Labels: []string{TransientTransactionError, NetworkError}}

               .          .    270: }

               .     3.53MB    271: wm, err = rw.ReadWireMessage(ctx)

               .          .    272: if err != nil {

               .          .    273: if _, ok := err.(Error); ok

      {          .          .    274: return nil, err          .          .    275: }

               .          .    276: // Connection errors are transient

               .          .    277: return nil, Error{Message: err.Error(), Labels: []string{TransientTransactionError, NetworkError}}

               .          .    278: }

               .          .    279:

               .          .    280: if r.Session != nil {

               .          .    281: err = r.Session.UpdateUseTime()

               .          .    282: if err != nil

      {          .          .    283: return nil, err          .          .    284: }

               .          .    285: }

               .    20.92MB    286: return r.Decode(desc, wm).Result()

               .          .    287:}

      (pprof) list Decode

      Total: 24.44MB

      ROUTINE ======================== go.mongodb.org/mongo-driver/x/network/command.(*Read).Decode in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/network/command/read.go

               0    20.92MB (flat, cum) 85.57% of Total

               .          .    221:func (r *Read) Decode(desc description.SelectedServer, wm wiremessage.WireMessage) *Read {

               .          .    222: switch wm.(type)

      {          .          .    223: case wiremessage.Reply:          .          .    224: r.decodeOpReply(wm)          .          .    225: default:          .    20.92MB    226: r.decodeOpMsg(wm)          .          .    227: }

               .          .    228:

               .          .    229: if r.err != nil {

               .          .    230: // decode functions set error if an invalid response document was returned or if the OK flag in the response was 0

               .          .    231: // if the OK flag was 0, a type Error is returned. otherwise, a special type is returned

      (pprof) cd decodeOpMsg

      Unrecognized command: "cd"

      (pprof) list decodeOpMsg

      Total: 24.44MB

      ROUTINE ======================== go.mongodb.org/mongo-driver/x/network/command.(*Read).decodeOpMsg in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/network/command/read.go

               0    20.92MB (flat, cum) 85.57% of Total

               .          .    180: if !ok

      {          .          .    181: r.err = fmt.Errorf("unsupported response wiremessage type %T", wm)          .          .    182: return          .          .    183: }

               .          .    184:

               .    20.92MB    185: r.result, r.err = decodeCommandOpMsg(msg)

               .          .    186:}

               .          .    187:

               .          .    188:func (r *Read) decodeOpReply(wm wiremessage.WireMessage) {

               .          .    189: reply, ok := wm.(wiremessage.Reply)

               .          .    190: if !ok {

      (pprof) peek decodeCommandOpMsg

      Showing nodes accounting for 25030.85kB, 100% of 25030.85kB total

      ---------------------------------------------------------+------------

            flat  flat%   sum%        cum   cum%   calls calls% + context   

      ---------------------------------------------------------+------------

                                              21419.71kB   100% |   go.mongodb.org/mongo-driver/x/network/command.(*Read).decodeOpMsg

               0     0%     0% 21419.71kB 85.57%                | go.mongodb.org/mongo-driver/x/network/command.decodeCommandOpMsg

                                              13847.71kB 64.65% |   go.mongodb.org/mongo-driver/x/bsonx.(*Doc).UnmarshalBSON

                                                  7572kB 35.35% |   go.mongodb.org/mongo-driver/x/bsonx.Doc.MarshalBSON

      ---------------------------------------------------------+------------

      (pprof)

       

       

       

        Attachments

          Activity

            People

            • Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: