memory still grows when program run

XMLWordPrintableJSON

    • Type: Bug
    • Resolution: Gone away
    • Priority: Major - P3
    • None
    • Affects Version/s: 1.0.1
    • Component/s: Core API
    • None
    • Environment:
      linux x86
      go version go1.12.4 darwin/amd64
    • None
    • None
    • None
    • None
    • None
    • None
    • None

      when i run a program that load 40MB data per 30 minitue, by profiling go program, i find the memory still grow, it not release to os

      i think it is a go-mongo bug, how to fix this issue?

      my code

      func UpdateTejia() {
      begin := common.GetMills()

      var buff bytes.Buffer

      myLog := mylog.GetMylog().Log()

      conf := GetConfigure()
      f := conf.Mongo.Tejia
      mongo := db.NewMongoOfficalClient(f.Host, f.Username, f.Password, f.Database, f.Authmechanism,
      f.Connect_timeout_s)
      err := mongo.Connect()
      if err != nil

      { buff.WriteString(fmt.Sprintf("%v", err)) myLog.Infof("%s", buff.String()) return }
      defer mongo.Close()
      client, _ := mongo.GetClientInfo()
      collection := client.Database(f.Database).Collection(f.Collection)

      //删除当前时间2天前的过期数据
      {
      ctx, cancle := context.WithTimeout(context.Background(), 120*time.Second)
      defer cancle()
      now := common.GetMills()
      timestamp := now - 2*24*3600*1000
      filter := bson.M{"inserttime": bson.M{"$lte": timestamp}}
      _, err = collection.DeleteMany(ctx, filter)
      if err != nil { buff.WriteString(fmt.Sprintf("%v", err)) myLog.Infof("%s", buff.String()) return }
      }
      //加载2天内更新的数据
      ctx, cancle := context.WithTimeout(context.Background(), 120*time.Second)
      defer cancle()
      cur, err := collection.Find(ctx, bson.D{})
      if err != nil { buff.WriteString(fmt.Sprintf("%v", err)) myLog.Infof("%s", buff.String()) return }

      defer cur.Close(ctx)

      var sli []MongoSpItem
      for cur.Next(ctx) {
      var result MongoSpItem
      err := cur.Decode(&result)
      if err != nil

      { continue }

      // do something with result....
      sli = append(sli, result)
      }
      if err = cur.Err(); err != nil

      { buff.WriteString(fmt.Sprintf("%v", err)) myLog.Infof("%s", buff.String()) return }

      //
      num := len(sli)
      t1 := common.GetMills()
      buff.WriteString(fmt.Sprintf("load %d data, use %d ms", num, t1-begin))
      myLog.Infof("%s", buff.String())

      if num > 0

      { mongoSpItem.sli = sli OtherHandleTejiaMongo() }

      }

      profile

      Saved profile in /Users/fredlee/pprof/pprof.tejia_index.alloc_objects.alloc_space.inuse_objects.inuse_space.010.pb.gz
      File: tejia_index
      Type: inuse_space
      Time: Apr 26, 2019 at 11:04am (CST)
      Entering interactive mode (type "help" for commands, "o" for options)
      (pprof) top -cum
      Showing nodes accounting for 2548.76MB, 71.40% of 3569.79MB total
      Dropped 54 nodes (cum <= 17.85MB)
      Showing top 10 nodes out of 21
      flat flat% sum% cum cum%
      0 0% 0% 1814.64MB 50.83% tejia_index/internal/async.Repeat.func1
      0 0% 0% 1814.64MB 50.83% tejia_index/internal/async.Repeat.func2
      1153.75MB 32.32% 32.32% 1814.64MB 50.83% tejia_index/util.UpdateTejia
      0 0% 32.32% 1428.12MB 40.01% runtime.mstart
      0 0% 32.32% 1428.12MB 40.01% runtime.newproc.func1
      0 0% 32.32% 1428.12MB 40.01% runtime.newproc1
      0 0% 32.32% 1428.12MB 40.01% runtime.systemstack
      1395.01MB 39.08% 71.40% 1395.01MB 39.08% runtime.malg
      0 0% 71.40% 659.01MB 18.46% go.mongodb.org/mongo-driver/bson.(*Decoder).Decode
      0 0% 71.40% 659.01MB 18.46% go.mongodb.org/mongo-driver/bson.UnmarshalWithRegistry
      (pprof) list UpdateTejia
      Total: 3.49GB
      ROUTINE ======================== tejia_index/util.UpdateTejia in /Users/fredlee/Documents/管家项目/huoli/tejia_index/util/mongo_tejia.go
      1.13GB 1.77GB (flat, cum) 50.83% of Total
      . . 206: defer cur.Close(ctx)
      . . 207:
      . . 208: var sli []MongoSpItem
      . . 209: for cur.Next(ctx) {
      . . 210: var result MongoSpItem
      . 659.01MB 211: err := cur.Decode(&result)
      . . 212: if err != nil

      { . . 213: continue . . 214: }

      . . 215: // do something with result....
      1.13GB 1.13GB 216: sli = append(sli, result)
      . . 217: }
      . . 218: if err = cur.Err(); err != nil

      { . . 219: buff.WriteString(fmt.Sprintf("%v", err)) . . 220: myLog.Infof("%s", buff.String()) . . 221: return . . 222: }

      . . 223:
      . . 224: //
      . . 225: num := len(sli)
      . . 226: t1 := common.GetMills()
      . . 227: buff.WriteString(fmt.Sprintf("load %d data, use %d ms", num, t1-begin))
      . . 228: myLog.Infof("%s", buff.String())
      . . 229:
      . . 230: if num > 0

      { . . 231: mongoSpItem.sli = sli . 1.87MB 232: OtherHandleTejiaMongo() . . 233: }

      . . 234:}
      . . 235:
      . . 236:func OtherHandleTejiaMongo() {
      . . 237: m := mongoSpItem
      (pprof) list Decode
      Total: 3.49GB
      ROUTINE ======================== encoding/json.(*Decoder).Decode in /usr/local/go/src/encoding/json/stream.go
      0 1MB (flat, cum) 0.028% of Total
      . . 58: if !dec.tokenValueAllowed() {
      . . 59: return &SyntaxError

      {msg: "not at beginning of value", Offset: dec.offset()}

      . . 60: }
      . . 61:
      . . 62: // Read whole value into buffer.
      . 513.31kB 63: n, err := dec.readValue()
      . . 64: if err != nil

      { . . 65: return err . . 66: }

      . . 67: dec.d.init(dec.buf[dec.scanp : dec.scanp+n])
      . . 68: dec.scanp += n
      . . 69:
      . . 70: // Don't save err from unmarshal into dec.err:
      . . 71: // the connection is still usable since we read a complete JSON
      . . 72: // object from it before the error happened.
      . 512.01kB 73: err = dec.d.unmarshal(v)
      . . 74:
      . . 75: // fixup token streaming state
      . . 76: dec.tokenValueEnd()
      . . 77:
      . . 78: return err
      ROUTINE ======================== encoding/json.(*Decoder).readValue in /usr/local/go/src/encoding/json/stream.go
      0 513.31kB (flat, cum) 0.014% of Total
      . . 129: dec.err = err
      . . 130: return 0, err
      . . 131: }
      . . 132:
      . . 133: n := scanp - dec.scanp
      . 513.31kB 134: err = dec.refill()
      . . 135: scanp = dec.scanp + n
      . . 136: }
      . . 137: return scanp - dec.scanp, nil
      . . 138:}
      . . 139:
      ROUTINE ======================== encoding/json.(*Decoder).refill in /usr/local/go/src/encoding/json/stream.go
      0 513.31kB (flat, cum) 0.014% of Total
      . . 154: copy(newBuf, dec.buf)
      . . 155: dec.buf = newBuf
      . . 156: }
      . . 157:
      . . 158: // Read. Delay error for next iteration (after scan).
      . 513.31kB 159: n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)])
      . . 160: dec.buf = dec.buf[0 : len(dec.buf)+n]
      . . 161:
      . . 162: return err
      . . 163:}
      . . 164:
      ROUTINE ======================== go.mongodb.org/mongo-driver/bson.(*Decoder).Decode in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/bson/decoder.go
      0 659.01MB (flat, cum) 18.46% of Total
      . . 81: rval = rval.Elem()
      . . 82: decoder, err := d.dc.LookupDecoder(rval.Type())
      . . 83: if err != nil

      { . . 84: return err . . 85: }

      . 659.01MB 86: return decoder.DecodeValue(d.dc, d.vr, rval)
      . . 87:}
      . . 88:
      . . 89:// Reset will reset the state of the decoder, using the same *DecodeContext used in
      . . 90:// the original construction but using vr for reading.
      . . 91:func (d *Decoder) Reset(vr bsonrw.ValueReader) error {
      ROUTINE ======================== go.mongodb.org/mongo-driver/bson/bsoncodec.(*StructCodec).DecodeValue in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/bson/bsoncodec/struct_codec.go
      0 771.02MB (flat, cum) 21.60% of Total
      . . 149: if err != nil

      { . . 150: return err . . 151: }

      . . 152:
      . . 153: for {
      . 241.50MB 154: name, vr, err := dr.ReadElement()
      . . 155: if err == bsonrw.ErrEOD

      { . . 156: break . . 157: }

      . . 158: if err != nil

      { . . 159: return err . . 160: }

      . . 161:
      . . 162: fd, exists := sd.fm[name]
      . . 163: if !exists {
      . . 164: if sd.inlineMap < 0 {
      . . 165: // The encoding/json package requires a flag to return on error for non-existent fields.
      . . 166: // This functionality seems appropriate for the struct codec.
      . . 167: err = vr.Skip()
      . . 168: if err != nil

      { . . 169: return err . . 170: }

      . . 171: continue
      . . 172: }
      . . 173:
      . . 174: elem := reflect.New(inlineMap.Type().Elem()).Elem()
      . . 175: err = decoder.DecodeValue(r, vr, elem)
      . . 176: if err != nil

      { . . 177: return err . . 178: }

      . . 179: inlineMap.SetMapIndex(reflect.ValueOf(name), elem)
      . . 180: continue
      . . 181: }
      . . 182:
      . . 183: var field reflect.Value
      . . 184: if fd.inline == nil

      { . . 185: field = val.Field(fd.idx) . . 186: }

      else

      { . . 187: field = val.FieldByIndex(fd.inline) . . 188: }

      . . 189:
      . . 190: if !field.CanSet()

      { // Being settable is a super set of being addressable. . . 191: return fmt.Errorf("cannot decode element '%s' into field %v; it is not settable", name, field) . . 192: }

      . . 193: if field.Kind() == reflect.Ptr && field.IsNil()

      { . . 194: field.Set(reflect.New(field.Type().Elem())) . . 195: }

      . . 196: field = field.Addr()
      . . 197:
      . . 198: dctx := DecodeContext

      {Registry: r.Registry, Truncate: fd.truncate}

      . . 199: if fd.decoder == nil {
      . . 200: return ErrNoDecoder

      {Type: field.Elem().Type()}

      . . 201: }
      . . 202:
      . . 203: if decoder, ok := fd.decoder.(ValueDecoder); ok {
      . 529.51MB 204: err = decoder.DecodeValue(dctx, vr, field.Elem())
      . . 205: if err != nil

      { . . 206: return err . . 207: }

      . . 208: continue
      . . 209: }
      ROUTINE ======================== go.mongodb.org/mongo-driver/bson/bsoncodec.DefaultValueDecoders.StringDecodeValue in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/bson/bsoncodec/default_value_decoders.go
      0 417.51MB (flat, cum) 11.70% of Total
      . . 309: var str string
      . . 310: var err error
      . . 311: switch vr.Type() {
      . . 312: // TODO(GODRIVER-577): Handle JavaScript and Symbol BSON types when allowed.
      . . 313: case bsontype.String:
      . 417.51MB 314: str, err = vr.ReadString()
      . . 315: if err != nil

      { . . 316: return err . . 317: }

      . . 318: default:
      . . 319: return fmt.Errorf("cannot decode %v into a string type", vr.Type())
      ROUTINE ======================== go.mongodb.org/mongo-driver/bson/bsoncodec.ValueDecoderFunc.DecodeValue in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/bson/bsoncodec/bsoncodec.go
      0 417.51MB (flat, cum) 11.70% of Total
      . . 151:// used as a ValueDecoder.
      . . 152:type ValueDecoderFunc func(DecodeContext, bsonrw.ValueReader, reflect.Value) error
      . . 153:
      . . 154:// DecodeValue implements the ValueDecoder interface.
      . . 155:func (fn ValueDecoderFunc) DecodeValue(dc DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error

      { . 417.51MB 156: return fn(dc, vr, val) . . 157:}

      . . 158:
      . . 159:// CodecZeroer is the interface implemented by Codecs that can also determine if
      . . 160:// a value of the type that would be encoded is zero.
      . . 161:type CodecZeroer interface

      { ROUTINE ======================== go.mongodb.org/mongo-driver/mongo.(*Cursor).Decode in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/mongo/cursor.go 0 659.01MB (flat, cum) 18.46% of Total . . 118: }

      . . 119:}
      . . 120:
      . . 121:// Decode will decode the current document into val.
      . . 122:func (c *Cursor) Decode(val interface{}) error

      { . 659.01MB 123: return bson.UnmarshalWithRegistry(c.registry, c.Current, val) . . 124:}

      . . 125:
      . . 126:// Err returns the current error.
      . . 127:func (c *Cursor) Err() error

      { return c.err }

      . . 128:
      (pprof)

            Assignee:
            Unassigned
            Reporter:
            orange.jackylee
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

              Created:
              Updated:
              Resolved: