XMLWordPrintableJSON

Details

    • Icon: Bug Bug
    • Resolution: Works as Designed
    • Icon: Major - P3 Major - P3
    • None
    • None
    • CRUD
    • None
    • Ubuntu 18.04

    Description

      I'm trying to get 10000 document at a time in mongodb

      Information :

      Code :

       

      package main
       
      import (
          "context"
          "fmt"
          "net/http"
          "os"
          "time"
       
          "go.mongodb.org/mongo-driver/bson"
          "go.mongodb.org/mongo-driver/mongo"
          "go.mongodb.org/mongo-driver/mongo/options"
      )
       
      var database *mongo.Database
       
      func main() {
       
          ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
          client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://20.20.20.43:27017"))
          if err != nil {
              panic(err)
          }
       
          database = client.Database("chat_data")
       
          chatText := make([]chat, 0)
          now := time.Now().Unix()
          ctx, _ = context.WithTimeout(context.Background(), 30*time.Second)
       
          // mongodb batch option
          opt := options.Find()
          opt.SetBatchSize(15_000)
          opt.SetAllowPartialResults(false)
       
          // mongodb filter
          filter := bson.M{"timestamp": bson.M{"$gte": now - 108000}}
       
          cur, err := database.Collection("chat").Find(ctx, filter, opt)
          if err != nil {
              // fmt.Fprint(w, err)
              fmt.Println(err)
              return
          }
          defer cur.Close(ctx)
       
          for cur.Next(ctx) {
              var result chat
              err := cur.Decode(&result)
              if err != nil {
                  fmt.Println(err)
                  continue
              }
              // do something with result....
              // fmt.Println(result)
              chatText = append(chatText, result)
          }
          if err := cur.Err(); err != nil {
              // fmt.Fprint(w, cur.Err())
              fmt.Println(err)
              return
          }
       
          fmt.Println("done")
          fmt.Println(len(chatText))
      }

      it's takes more than 30 second to only get the full result, while pymongo only need 0m2.159s for 36k document (with same filter)

       

       

      Attachments

        Activity

          People

            Unassigned Unassigned
            suyatno.hardi@gmail.com Hardi N/A
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: