Details
-
Bug
-
Resolution: Works as Designed
-
Major - P3
-
None
-
None
-
None
-
Ubuntu 18.04
Description
I'm trying to get 10000 document at a time in mongodb
Information :
- Driver https://github.com/mongodb/mongo-go-driver
- opt.SetBatchSize(15_000)
- opt.SetAllowPartialResults(false)
- index on timestamp
Code :
package main |
|
|
import ( |
"context" |
"fmt" |
"net/http" |
"os" |
"time" |
|
|
"go.mongodb.org/mongo-driver/bson" |
"go.mongodb.org/mongo-driver/mongo" |
"go.mongodb.org/mongo-driver/mongo/options" |
)
|
|
|
var database *mongo.Database
|
|
|
func main() {
|
|
|
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) |
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://20.20.20.43:27017")) |
if err != nil { |
panic(err)
|
}
|
|
|
database = client.Database("chat_data") |
|
|
chatText := make([]chat, 0) |
now := time.Now().Unix()
|
ctx, _ = context.WithTimeout(context.Background(), 30*time.Second) |
|
|
// mongodb batch option |
opt := options.Find()
|
opt.SetBatchSize(15_000)
|
opt.SetAllowPartialResults(false) |
|
|
// mongodb filter |
filter := bson.M{"timestamp": bson.M{"$gte": now - 108000}} |
|
|
cur, err := database.Collection("chat").Find(ctx, filter, opt) |
if err != nil { |
// fmt.Fprint(w, err) |
fmt.Println(err)
|
return |
}
|
defer cur.Close(ctx)
|
|
|
for cur.Next(ctx) { |
var result chat
|
err := cur.Decode(&result)
|
if err != nil { |
fmt.Println(err)
|
continue |
}
|
// do something with result.... |
// fmt.Println(result) |
chatText = append(chatText, result)
|
}
|
if err := cur.Err(); err != nil { |
// fmt.Fprint(w, cur.Err()) |
fmt.Println(err)
|
return |
}
|
|
|
fmt.Println("done") |
fmt.Println(len(chatText))
|
}
|
it's takes more than 30 second to only get the full result, while pymongo only need 0m2.159s for 36k document (with same filter)