-
Type:
Bug
-
Resolution: Gone away
-
Priority:
Major - P3
-
None
-
Affects Version/s: 1.0.1
-
Component/s: Core API
-
None
-
Environment:linux x86
go version go1.12.4 darwin/amd64
-
None
-
None
-
None
-
None
-
None
-
None
-
None
when i run a program that load 40MB data per 30 minitue, by profiling go program, i find the memory still grow, it not release to os
i think it is a go-mongo bug, how to fix this issue?
my code
func UpdateTejia() {
begin := common.GetMills()
var buff bytes.Buffer
myLog := mylog.GetMylog().Log()
conf := GetConfigure()
f := conf.Mongo.Tejia
mongo := db.NewMongoOfficalClient(f.Host, f.Username, f.Password, f.Database, f.Authmechanism,
f.Connect_timeout_s)
err := mongo.Connect()
if err != nil
defer mongo.Close()
client, _ := mongo.GetClientInfo()
collection := client.Database(f.Database).Collection(f.Collection)
//删除当前时间2天前的过期数据
{
ctx, cancle := context.WithTimeout(context.Background(), 120*time.Second)
defer cancle()
now := common.GetMills()
timestamp := now - 2*24*3600*1000
filter := bson.M{"inserttime": bson.M{"$lte": timestamp}}
_, err = collection.DeleteMany(ctx, filter)
if err != nil { buff.WriteString(fmt.Sprintf("%v", err)) myLog.Infof("%s", buff.String()) return }
}
//加载2天内更新的数据
ctx, cancle := context.WithTimeout(context.Background(), 120*time.Second)
defer cancle()
cur, err := collection.Find(ctx, bson.D{})
if err != nil { buff.WriteString(fmt.Sprintf("%v", err)) myLog.Infof("%s", buff.String()) return }
defer cur.Close(ctx)
var sli []MongoSpItem
for cur.Next(ctx) {
var result MongoSpItem
err := cur.Decode(&result)
if err != nil
// do something with result....
sli = append(sli, result)
}
if err = cur.Err(); err != nil
//
num := len(sli)
t1 := common.GetMills()
buff.WriteString(fmt.Sprintf("load %d data, use %d ms", num, t1-begin))
myLog.Infof("%s", buff.String())
if num > 0
{ mongoSpItem.sli = sli OtherHandleTejiaMongo() }}
profile
Saved profile in /Users/fredlee/pprof/pprof.tejia_index.alloc_objects.alloc_space.inuse_objects.inuse_space.010.pb.gz
File: tejia_index
Type: inuse_space
Time: Apr 26, 2019 at 11:04am (CST)
Entering interactive mode (type "help" for commands, "o" for options)
(pprof) top -cum
Showing nodes accounting for 2548.76MB, 71.40% of 3569.79MB total
Dropped 54 nodes (cum <= 17.85MB)
Showing top 10 nodes out of 21
flat flat% sum% cum cum%
0 0% 0% 1814.64MB 50.83% tejia_index/internal/async.Repeat.func1
0 0% 0% 1814.64MB 50.83% tejia_index/internal/async.Repeat.func2
1153.75MB 32.32% 32.32% 1814.64MB 50.83% tejia_index/util.UpdateTejia
0 0% 32.32% 1428.12MB 40.01% runtime.mstart
0 0% 32.32% 1428.12MB 40.01% runtime.newproc.func1
0 0% 32.32% 1428.12MB 40.01% runtime.newproc1
0 0% 32.32% 1428.12MB 40.01% runtime.systemstack
1395.01MB 39.08% 71.40% 1395.01MB 39.08% runtime.malg
0 0% 71.40% 659.01MB 18.46% go.mongodb.org/mongo-driver/bson.(*Decoder).Decode
0 0% 71.40% 659.01MB 18.46% go.mongodb.org/mongo-driver/bson.UnmarshalWithRegistry
(pprof) list UpdateTejia
Total: 3.49GB
ROUTINE ======================== tejia_index/util.UpdateTejia in /Users/fredlee/Documents/管家项目/huoli/tejia_index/util/mongo_tejia.go
1.13GB 1.77GB (flat, cum) 50.83% of Total
. . 206: defer cur.Close(ctx)
. . 207:
. . 208: var sli []MongoSpItem
. . 209: for cur.Next(ctx) {
. . 210: var result MongoSpItem
. 659.01MB 211: err := cur.Decode(&result)
. . 212: if err != nil
. . 215: // do something with result....
1.13GB 1.13GB 216: sli = append(sli, result)
. . 217: }
. . 218: if err = cur.Err(); err != nil
. . 223:
. . 224: //
. . 225: num := len(sli)
. . 226: t1 := common.GetMills()
. . 227: buff.WriteString(fmt.Sprintf("load %d data, use %d ms", num, t1-begin))
. . 228: myLog.Infof("%s", buff.String())
. . 229:
. . 230: if num > 0
. . 234:}
. . 235:
. . 236:func OtherHandleTejiaMongo() {
. . 237: m := mongoSpItem
(pprof) list Decode
Total: 3.49GB
ROUTINE ======================== encoding/json.(*Decoder).Decode in /usr/local/go/src/encoding/json/stream.go
0 1MB (flat, cum) 0.028% of Total
. . 58: if !dec.tokenValueAllowed() {
. . 59: return &SyntaxError
. . 60: }
. . 61:
. . 62: // Read whole value into buffer.
. 513.31kB 63: n, err := dec.readValue()
. . 64: if err != nil
. . 67: dec.d.init(dec.buf[dec.scanp : dec.scanp+n])
. . 68: dec.scanp += n
. . 69:
. . 70: // Don't save err from unmarshal into dec.err:
. . 71: // the connection is still usable since we read a complete JSON
. . 72: // object from it before the error happened.
. 512.01kB 73: err = dec.d.unmarshal(v)
. . 74:
. . 75: // fixup token streaming state
. . 76: dec.tokenValueEnd()
. . 77:
. . 78: return err
ROUTINE ======================== encoding/json.(*Decoder).readValue in /usr/local/go/src/encoding/json/stream.go
0 513.31kB (flat, cum) 0.014% of Total
. . 129: dec.err = err
. . 130: return 0, err
. . 131: }
. . 132:
. . 133: n := scanp - dec.scanp
. 513.31kB 134: err = dec.refill()
. . 135: scanp = dec.scanp + n
. . 136: }
. . 137: return scanp - dec.scanp, nil
. . 138:}
. . 139:
ROUTINE ======================== encoding/json.(*Decoder).refill in /usr/local/go/src/encoding/json/stream.go
0 513.31kB (flat, cum) 0.014% of Total
. . 154: copy(newBuf, dec.buf)
. . 155: dec.buf = newBuf
. . 156: }
. . 157:
. . 158: // Read. Delay error for next iteration (after scan).
. 513.31kB 159: n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)])
. . 160: dec.buf = dec.buf[0 : len(dec.buf)+n]
. . 161:
. . 162: return err
. . 163:}
. . 164:
ROUTINE ======================== go.mongodb.org/mongo-driver/bson.(*Decoder).Decode in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/bson/decoder.go
0 659.01MB (flat, cum) 18.46% of Total
. . 81: rval = rval.Elem()
. . 82: decoder, err := d.dc.LookupDecoder(rval.Type())
. . 83: if err != nil
. 659.01MB 86: return decoder.DecodeValue(d.dc, d.vr, rval)
. . 87:}
. . 88:
. . 89:// Reset will reset the state of the decoder, using the same *DecodeContext used in
. . 90:// the original construction but using vr for reading.
. . 91:func (d *Decoder) Reset(vr bsonrw.ValueReader) error {
ROUTINE ======================== go.mongodb.org/mongo-driver/bson/bsoncodec.(*StructCodec).DecodeValue in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/bson/bsoncodec/struct_codec.go
0 771.02MB (flat, cum) 21.60% of Total
. . 149: if err != nil
. . 152:
. . 153: for {
. 241.50MB 154: name, vr, err := dr.ReadElement()
. . 155: if err == bsonrw.ErrEOD
. . 158: if err != nil
{ . . 159: return err . . 160: } . . 161:
. . 162: fd, exists := sd.fm[name]
. . 163: if !exists {
. . 164: if sd.inlineMap < 0 {
. . 165: // The encoding/json package requires a flag to return on error for non-existent fields.
. . 166: // This functionality seems appropriate for the struct codec.
. . 167: err = vr.Skip()
. . 168: if err != nil
. . 171: continue
. . 172: }
. . 173:
. . 174: elem := reflect.New(inlineMap.Type().Elem()).Elem()
. . 175: err = decoder.DecodeValue(r, vr, elem)
. . 176: if err != nil
. . 179: inlineMap.SetMapIndex(reflect.ValueOf(name), elem)
. . 180: continue
. . 181: }
. . 182:
. . 183: var field reflect.Value
. . 184: if fd.inline == nil
else
{ . . 187: field = val.FieldByIndex(fd.inline) . . 188: } . . 189:
. . 190: if !field.CanSet()
. . 193: if field.Kind() == reflect.Ptr && field.IsNil()
{ . . 194: field.Set(reflect.New(field.Type().Elem())) . . 195: } . . 196: field = field.Addr()
. . 197:
. . 198: dctx := DecodeContext
. . 199: if fd.decoder == nil {
. . 200: return ErrNoDecoder
. . 201: }
. . 202:
. . 203: if decoder, ok := fd.decoder.(ValueDecoder); ok {
. 529.51MB 204: err = decoder.DecodeValue(dctx, vr, field.Elem())
. . 205: if err != nil
. . 208: continue
. . 209: }
ROUTINE ======================== go.mongodb.org/mongo-driver/bson/bsoncodec.DefaultValueDecoders.StringDecodeValue in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/bson/bsoncodec/default_value_decoders.go
0 417.51MB (flat, cum) 11.70% of Total
. . 309: var str string
. . 310: var err error
. . 311: switch vr.Type() {
. . 312: // TODO(GODRIVER-577): Handle JavaScript and Symbol BSON types when allowed.
. . 313: case bsontype.String:
. 417.51MB 314: str, err = vr.ReadString()
. . 315: if err != nil
. . 318: default:
. . 319: return fmt.Errorf("cannot decode %v into a string type", vr.Type())
ROUTINE ======================== go.mongodb.org/mongo-driver/bson/bsoncodec.ValueDecoderFunc.DecodeValue in /Users/fredlee/Documents/develop/go/workspace/pkg/mod/go.mongodb.org/mongo-driver@v1.0.1/bson/bsoncodec/bsoncodec.go
0 417.51MB (flat, cum) 11.70% of Total
. . 151:// used as a ValueDecoder.
. . 152:type ValueDecoderFunc func(DecodeContext, bsonrw.ValueReader, reflect.Value) error
. . 153:
. . 154:// DecodeValue implements the ValueDecoder interface.
. . 155:func (fn ValueDecoderFunc) DecodeValue(dc DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error
. . 158:
. . 159:// CodecZeroer is the interface implemented by Codecs that can also determine if
. . 160:// a value of the type that would be encoded is zero.
. . 161:type CodecZeroer interface
. . 119:}
. . 120:
. . 121:// Decode will decode the current document into val.
. . 122:func (c *Cursor) Decode(val interface{}) error
. . 125:
. . 126:// Err returns the current error.
. . 127:func (c *Cursor) Err() error
. . 128:
(pprof)