Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-35063

Multiple concurrent splitVector on same chunk

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Minor - P4
    • Resolution: Done
    • Affects Version/s: 3.4.14
    • Fix Version/s: None
    • Component/s: Sharding
    • Labels:
      None
    • Operating System:
      ALL
    • Steps To Reproduce:
      Hide

      I don't know exact reproducing scenario.

       

      We use 15 shard cluster,

      and doing concurrent insert & update & delete and find operations.

      And collections are sharded with hash.

      Show
      I don't know exact reproducing scenario.   We use 15 shard cluster, and doing concurrent insert & update & delete and find operations. And collections are sharded with hash.

      Description

      Multiple splitVector commands run on the same chunk at the same time.

      These gang of splitVector threads takes some cpu resources and make linux "load average" high. Sometimes, concurrent splitVector threads are more than below capture.

       

      Does splitVector has something to do with Client Driver ?

       

      PRIMARY> db.currentOp({secs_running:{$gt:50}})
      {
        "inprog" : [
          {
            "desc" : "conn734364",
            "threadId" : "139976066397952",
            "connectionId" : 734364,
            "client" : "xx.xx.xx.147:39482",
            "clientMetadata" : {
              "driver" : {
                "name" : "PyMongo",
                "version" : "3.5.1"
              },
              "os" : {
                "type" : "Linux",
                "name" : "CentOS Linux 7.2.1511 Core",
                "architecture" : "x86_64",
                "version" : "3.10.0-327.22.2.el7.x86_64"
              },
              "platform" : "CPython 2.7.5.final.0"
            },
            "active" : true,
            "opid" : -1800224412,
            "secs_running" : 118,
            "microsecs_running" : NumberLong(118036322),
            "op" : "command",
            "ns" : "admin.$cmd",
            "query" : {
              "splitVector" : "mydb.mycoll",
              "keyPattern" : {
                "fd_shardkey" : "hashed"
              },
              "min" : {
                "fd_shardkey" : NumberLong("1585919574544073309")
              },
              "max" : {
                "fd_shardkey" : NumberLong("1585925300908251118")
              },
              "maxChunkSizeBytes" : NumberLong(67108864)
            },
            "numYields" : 422189,
            "locks" : {
              "Global" : "r",
              "Database" : "r",
              "Collection" : "r"
            },
            "waitingForLock" : false,
            "lockStats" : {
              "Global" : {
                "acquireCount" : {
                  "r" : NumberLong(844380)
                }
              },
              "Database" : {
                "acquireCount" : {
                  "r" : NumberLong(422190)
                },
                "acquireWaitCount" : {
                  "r" : NumberLong(1)
                },
                "timeAcquiringMicros" : {
                  "r" : NumberLong(94)
                }
              },
              "Collection" : {
                "acquireCount" : {
                  "r" : NumberLong(422190)
                }
              }
            }
          },
          {
            "desc" : "conn734226",
            "threadId" : "139974009476864",
            "connectionId" : 734226,
            "client" : "xx.xx.xx.147:37786",
            "clientMetadata" : {
              "driver" : {
                "name" : "PyMongo",
                "version" : "3.5.1"
              },
              "os" : {
                "type" : "Linux",
                "name" : "CentOS Linux 7.2.1511 Core",
                "architecture" : "x86_64",
                "version" : "3.10.0-327.22.2.el7.x86_64"
              },
              "platform" : "CPython 2.7.5.final.0"
            },
            "active" : true,
            "opid" : -1800224471,
            "secs_running" : 118,
            "microsecs_running" : NumberLong(118048371),
            "op" : "command",
            "ns" : "admin.$cmd",
            "query" : {
              "splitVector" : "mydb.mycoll",
              "keyPattern" : {
                "fd_shardkey" : "hashed"
              },
              "min" : {
                "fd_shardkey" : NumberLong("1585919574544073309")
              },
              "max" : {
                "fd_shardkey" : NumberLong("1585925300908251118")
              },
              "maxChunkSizeBytes" : NumberLong(67108864)
            },
            "numYields" : 399251,
            "locks" : {
              "Global" : "r",
              "Database" : "r",
              "Collection" : "r"
            },
            "waitingForLock" : false,
            "lockStats" : {
              "Global" : {
                "acquireCount" : {
                  "r" : NumberLong(798504)
                }
              },
              "Database" : {
                "acquireCount" : {
                  "r" : NumberLong(399252)
                },
                "acquireWaitCount" : {
                  "r" : NumberLong(1)
                },
                "timeAcquiringMicros" : {
                  "r" : NumberLong(158)
                }
              },
              "Collection" : {
                "acquireCount" : {
                  "r" : NumberLong(399252)
                }
              }
            }
          },
          {
            "desc" : "conn734224",
            "threadId" : "139976032712448",
            "connectionId" : 734224,
            "client" : "xx.xx.xx.147:37782",
            "clientMetadata" : {
              "driver" : {
                "name" : "PyMongo",
                "version" : "3.5.1"
              },
              "os" : {
                "type" : "Linux",
                "name" : "CentOS Linux 7.2.1511 Core",
                "architecture" : "x86_64",
                "version" : "3.10.0-327.22.2.el7.x86_64"
              },
              "platform" : "CPython 2.7.5.final.0"
            },
            "active" : true,
            "opid" : -1800226779,
            "secs_running" : 118,
            "microsecs_running" : NumberLong(118400690),
            "op" : "command",
            "ns" : "admin.$cmd",
            "query" : {
              "splitVector" : "mydb.mycoll",
              "keyPattern" : {
                "fd_shardkey" : "hashed"
              },
              "min" : {
                "fd_shardkey" : NumberLong("1585919574544073309")
              },
              "max" : {
                "fd_shardkey" : NumberLong("1585925300908251118")
              },
              "maxChunkSizeBytes" : NumberLong(67108864)
            },
            "numYields" : 416562,
            "locks" : {
              "Global" : "r",
              "Database" : "r",
              "Collection" : "r"
            },
            "waitingForLock" : false,
            "lockStats" : {
              "Global" : {
                "acquireCount" : {
                  "r" : NumberLong(833126)
                }
              },
              "Database" : {
                "acquireCount" : {
                  "r" : NumberLong(416563)
                },
                "acquireWaitCount" : {
                  "r" : NumberLong(1)
                },
                "timeAcquiringMicros" : {
                  "r" : NumberLong(200)
                }
              },
              "Collection" : {
                "acquireCount" : {
                  "r" : NumberLong(416563)
                }
              }
            }
          },
          {
            "desc" : "conn734365",
            "threadId" : "139976806426368",
            "connectionId" : 734365,
            "client" : "xx.xx.xx.147:39492",
            "clientMetadata" : {
              "driver" : {
                "name" : "PyMongo",
                "version" : "3.5.1"
              },
              "os" : {
                "type" : "Linux",
                "name" : "CentOS Linux 7.2.1511 Core",
                "architecture" : "x86_64",
                "version" : "3.10.0-327.22.2.el7.x86_64"
              },
              "platform" : "CPython 2.7.5.final.0"
            },
            "active" : true,
            "opid" : -1800222592,
            "secs_running" : 117,
            "microsecs_running" : NumberLong(117677575),
            "op" : "command",
            "ns" : "admin.$cmd",
            "query" : {
              "splitVector" : "mydb.mycoll",
              "keyPattern" : {
                "fd_shardkey" : "hashed"
              },
              "min" : {
                "fd_shardkey" : NumberLong("1585919574544073309")
              },
              "max" : {
                "fd_shardkey" : NumberLong("1585925300908251118")
              },
              "maxChunkSizeBytes" : NumberLong(67108864)
            },
            "numYields" : 418779,
            "locks" : {
              "Global" : "r",
              "Database" : "r",
              "Collection" : "r"
            },
            "waitingForLock" : false,
            "lockStats" : {
              "Global" : {
                "acquireCount" : {
                  "r" : NumberLong(837560)
                }
              },
              "Database" : {
                "acquireCount" : {
                  "r" : NumberLong(418780)
                },
                "acquireWaitCount" : {
                  "r" : NumberLong(1)
                },
                "timeAcquiringMicros" : {
                  "r" : NumberLong(206)
                }
              },
              "Collection" : {
                "acquireCount" : {
                  "r" : NumberLong(418780)
                }
              }
            }
          },
          {
            "desc" : "conn734227",
            "threadId" : "139976726423296",
            "connectionId" : 734227,
            "client" : "xx.xx.xx.147:37788",
            "clientMetadata" : {
              "driver" : {
                "name" : "PyMongo",
                "version" : "3.5.1"
              },
              "os" : {
                "type" : "Linux",
                "name" : "CentOS Linux 7.2.1511 Core",
                "architecture" : "x86_64",
                "version" : "3.10.0-327.22.2.el7.x86_64"
              },
              "platform" : "CPython 2.7.5.final.0"
            },
            "active" : true,
            "opid" : -1800224505,
            "secs_running" : 118,
            "microsecs_running" : NumberLong(118054608),
            "op" : "command",
            "ns" : "admin.$cmd",
            "query" : {
              "splitVector" : "mydb.mycoll",
              "keyPattern" : {
                "fd_shardkey" : "hashed"
              },
              "min" : {
                "fd_shardkey" : NumberLong("1585919574544073309")
              },
              "max" : {
                "fd_shardkey" : NumberLong("1585925300908251118")
              },
              "maxChunkSizeBytes" : NumberLong(67108864)
            },
            "numYields" : 390945,
            "locks" : {
              "Global" : "r",
              "Database" : "r",
              "Collection" : "r"
            },
            "waitingForLock" : false,
            "lockStats" : {
              "Global" : {
                "acquireCount" : {
                  "r" : NumberLong(781892)
                }
              },
              "Database" : {
                "acquireCount" : {
                  "r" : NumberLong(390946)
                },
                "acquireWaitCount" : {
                  "r" : NumberLong(1)
                },
                "timeAcquiringMicros" : {
                  "r" : NumberLong(198)
                }
              },
              "Collection" : {
                "acquireCount" : {
                  "r" : NumberLong(390946)
                }
              }
            }
          }
        ],
        "ok" : 1
      }
      

        Attachments

          Activity

            People

            Assignee:
            backlog-server-triage Backlog - Triage Team
            Reporter:
            matt.lee Matt SeongUck Lee
            Participants:
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

              Dates

              Created:
              Updated:
              Resolved: