[GODRIVER-1138] size of collection is 3.6MB, load from mongo with v1.0.3, profile it use 26MB Created: 14/Jun/19 Updated: 27/Oct/23 Resolved: 11/Jul/19 |
|
| Status: | Closed |
| Project: | Go Driver |
| Component/s: | CRUD |
| Affects Version/s: | 1.0.3 |
| Fix Version/s: | None |
| Type: | Improvement | Priority: | Major - P3 |
| Reporter: | orange.jackylee | Assignee: | Kristofer Brandow (Inactive) |
| Resolution: | Gone away | Votes: | 0 |
| Labels: | None | ||
| Remaining Estimate: | Not Specified | ||
| Time Spent: | Not Specified | ||
| Original Estimate: | Not Specified | ||
| Environment: |
go version go1.12.5 |
||
| Description |
|
size of collection is 3.6MB { , , , , , , , , , , }, ,
mycode .............. collection := client.Database(mgoc.Database).Collection(GetGlobalAirportCollection().Collection) collection := client.Database(mgoc.Database).Collection(GetGlobalAirportCollection().Collection) ctx, cancle := context.WithTimeout(context.Background(), 30*time.Second) defer cancle() cur, err := collection.Find(ctx, bson.D{}) if err != nil { return } defer cur.Close(ctx) for cur.Next(ctx) { var result GlobAirport err := cur.Decode(&result) if err != nil { continue }// do something with result.... sli = append(sli, result) } if err = cur.Err(); err != nil { return } ..............
go profile macbookpro:Downloads fredlee$ go tool pprof http://ip:port/debug/pprof/heap Fetching profile over HTTP from http://ip:port/debug/pprof/heap Saved profile in /Users/fredlee/pprof/pprof.catweb_server.alloc_objects.alloc_space.inuse_objects.inuse_space.009.pb.gz File: catweb_server Type: inuse_space Time: Jun 14, 2019 at 8:55pm (CST) Entering interactive mode (type "help" for commands, "o" for options) (pprof) top10 -cum Showing nodes accounting for 0, 0% of 24.44MB total Showing top 10 nodes out of 40 flat flat% sum% cum cum% 0 0% 0% 24.44MB 100% catweb_server/internal/async.(*M1).Start 0 0% 0% 24.44MB 100% catweb_server/internal/async.(*M1).handle 0 0% 0% 24.44MB 100% catweb_server/internal/db.(*MongoOfficalClient).Do 0 0% 0% 24.44MB 100% catweb_server/util.(*GlobAirports).LoadFromDb 0 0% 0% 24.44MB 100% catweb_server/util.(*GlobAirports).LoadFromDb.func1 0 0% 0% 24.44MB 100% go.mongodb.org/mongo-driver/mongo.(*Cursor).Next 0 0% 0% 24.44MB 100% go.mongodb.org/mongo-driver/x/mongo/driver.(*BatchCursor).Next 0 0% 0% 24.44MB 100% go.mongodb.org/mongo-driver/x/mongo/driver.(*BatchCursor).getMore 0 0% 0% 24.44MB 100% go.mongodb.org/mongo-driver/x/network/command.(*GetMore).RoundTrip 0 0% 0% 24.44MB 100% go.mongodb.org/mongo-driver/x/network/command.(*Read).RoundTrip (pprof) list LoadFromDb Total: 24.44MB ROUTINE ======================== catweb_server/util.(*GlobAirports).LoadFromDb in /Users/fredlee/Documents/project/商城/catweb_server/util/mongo_globalairport.go 0 24.44MB (flat, cum) 100% of Total . . 71: if err = cur.Err(); err != nil { . . 72: return . . 73: }. . 74: return . . 75: } . 24.44MB 76: err = mgoc.Do(handler) . . 77: if err != nil { . . 78: return . . 79: }. . 80: . . 81: num = len(sli) ROUTINE ======================== catweb_server/util.(*GlobAirports).LoadFromDb.func1 in /Users/fredlee/Documents/project/商城/catweb_server/util/mongo_globalairport.go 0 24.44MB (flat, cum) 100% of Total . . 57: cur, err := collection.Find(ctx, bson.D{}) . . 58: if err != nil { . . 59: return . . 60: }. . 61: defer cur.Close(ctx) . 24.44MB 62: for cur.Next(ctx) { . . 63: var result GlobAirport . . 64: err := cur.Decode(&result) . . 65: if err != nil { . . 66: continue . . 67: }(pprof) list Next Total: 24.44MB ROUTINE ======================== go.mongodb.org/mongo-driver/mongo.(*Cursor).Next in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/mongo/cursor.go 0 24.44MB (flat, cum) 100% of Total . . 88: . . 89: // call the Next method in a loop until at least one document is returned in the next batch or . . 90: // the context times out. . . 91: for { . . 92: // If we don't have a next batch . 24.44MB 93: if !c.bc.Next(ctx) { . . 94: // Do we have an error? If so we return false. . . 95: c.err = c.bc.Err() . . 96: if c.err != nil { . . 97: return false . . 98: }ROUTINE ======================== go.mongodb.org/mongo-driver/x/mongo/driver.(*BatchCursor).Next in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/mongo/driver/batch_cursor.go 0 24.44MB (flat, cum) 100% of Total . . 155: } . . 156: . . 157: if bc.legacy() { . . 158: bc.legacyGetMore(ctx) . . 159: }else { . 24.44MB 160: bc.getMore(ctx) . . 161: }. . 162: . . 163: switch bc.currentBatch.Style { . . 164: case bsoncore.SequenceStyle: . . 165: return len(bc.currentBatch.Data) > 0 (pprof) list getMore Total: 24.44MB ROUTINE ======================== go.mongodb.org/mongo-driver/x/mongo/driver.(*BatchCursor).getMore in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/mongo/driver/batch_cursor.go 0 24.44MB (flat, cum) 100% of Total . . 244: Clock: bc.clock, . . 245: ID: bc.id, . . 246: NS: bc.namespace, . . 247: Opts: bc.opts, . . 248: Session: bc.clientSession, . 24.44MB 249: }).RoundTrip(ctx, bc.server.SelectedDescription(), conn) . . 250: if err != nil { . . 251: _ = conn.Close() // The command response error is more important here . . 252: bc.err = err . . 253: return . . 254: }(pprof) list RoundTrip Total: 24.44MB ROUTINE ======================== go.mongodb.org/mongo-driver/x/network/command.(*GetMore).RoundTrip in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/network/command/get_more.go 0 24.44MB (flat, cum) 100% of Total . . 97: cmd, err := gm.encode(desc) . . 98: if err != nil { . . 99: return nil, err . . 100: }. . 101: . 24.44MB 102: rdr, err := cmd.RoundTrip(ctx, desc, rw) . . 103: if err != nil { . . 104: return nil, err . . 105: }. . 106: . . 107: return gm.decode(desc, rdr).Result() ROUTINE ======================== go.mongodb.org/mongo-driver/x/network/command.(*Read).RoundTrip in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/network/command/read.go 0 24.44MB (flat, cum) 100% of Total . . 266: return nil, err . . 267: } . . 268: // Connection errors are transient . . 269: return nil, Error{Message: err.Error(), Labels: []string{TransientTransactionError, NetworkError}} . . 270: } . 3.53MB 271: wm, err = rw.ReadWireMessage(ctx) . . 272: if err != nil { . . 273: if _, ok := err.(Error); ok { . . 274: return nil, err . . 275: }. . 276: // Connection errors are transient . . 277: return nil, Error{Message: err.Error(), Labels: []string{TransientTransactionError, NetworkError}} . . 278: } . . 279: . . 280: if r.Session != nil { . . 281: err = r.Session.UpdateUseTime() . . 282: if err != nil { . . 283: return nil, err . . 284: }. . 285: } . 20.92MB 286: return r.Decode(desc, wm).Result() . . 287:} (pprof) list Decode Total: 24.44MB ROUTINE ======================== go.mongodb.org/mongo-driver/x/network/command.(*Read).Decode in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/network/command/read.go 0 20.92MB (flat, cum) 85.57% of Total . . 221:func (r *Read) Decode(desc description.SelectedServer, wm wiremessage.WireMessage) *Read { . . 222: switch wm.(type) { . . 223: case wiremessage.Reply: . . 224: r.decodeOpReply(wm) . . 225: default: . 20.92MB 226: r.decodeOpMsg(wm) . . 227: }. . 228: . . 229: if r.err != nil { . . 230: // decode functions set error if an invalid response document was returned or if the OK flag in the response was 0 . . 231: // if the OK flag was 0, a type Error is returned. otherwise, a special type is returned (pprof) cd decodeOpMsg Unrecognized command: "cd" (pprof) list decodeOpMsg Total: 24.44MB ROUTINE ======================== go.mongodb.org/mongo-driver/x/network/command.(*Read).decodeOpMsg in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.3/x/network/command/read.go 0 20.92MB (flat, cum) 85.57% of Total . . 180: if !ok { . . 181: r.err = fmt.Errorf("unsupported response wiremessage type %T", wm) . . 182: return . . 183: }. . 184: . 20.92MB 185: r.result, r.err = decodeCommandOpMsg(msg) . . 186:} . . 187: . . 188:func (r *Read) decodeOpReply(wm wiremessage.WireMessage) { . . 189: reply, ok := wm.(wiremessage.Reply) . . 190: if !ok { (pprof) peek decodeCommandOpMsg Showing nodes accounting for 25030.85kB, 100% of 25030.85kB total --------------------------------------------------------- flat flat% sum% cum cum% calls calls% + context --------------------------------------------------------- 21419.71kB 100% | go.mongodb.org/mongo-driver/x/network/command.(*Read).decodeOpMsg 0 0% 0% 21419.71kB 85.57% | go.mongodb.org/mongo-driver/x/network/command.decodeCommandOpMsg 13847.71kB 64.65% | go.mongodb.org/mongo-driver/x/bsonx.(*Doc).UnmarshalBSON 7572kB 35.35% | go.mongodb.org/mongo-driver/x/bsonx.Doc.MarshalBSON --------------------------------------------------------- (pprof)
|
| Comments |
| Comment by Esha Bhargava [ 20/Jun/19 ] |
|
jackylee Can you please try to reproduce this on the master branch? This code path has changed significantly and will no longer be used. |
| Comment by orange.jackylee [ 18/Jun/19 ] |
|
why not reply my issue |