Loading...

Type: Bug
Resolution: Done
Priority: Minor - P4
Fix Version/s: None
Affects Version/s: 3.4.14
Component/s: Sharding
Labels:
None

Assigned Teams:

Server Triage
Operating System:
ALL
Steps To Reproduce:

Hide

I don't know exact reproducing scenario.

We use 15 shard cluster,

and doing concurrent insert & update & delete and find operations.

And collections are sharded with hash.

Show
I don't know exact reproducing scenario. We use 15 shard cluster, and doing concurrent insert & update & delete and find operations. And collections are sharded with hash.
CAR Domain/s:
None

Aha! Reference:
None
Tracking Level:
None
Risk Status:
None
Exec Notes:
None
Goal Name(s):
None
Goal Link:
None

Multiple splitVector commands run on the same chunk at the same time.

These gang of splitVector threads takes some cpu resources and make linux "load average" high. Sometimes, concurrent splitVector threads are more than below capture.

Does splitVector has something to do with Client Driver ?

PRIMARY> db.currentOp({secs_running:{$gt:50}})
{
  "inprog" : [
    {
      "desc" : "conn734364",
      "threadId" : "139976066397952",
      "connectionId" : 734364,
      "client" : "xx.xx.xx.147:39482",
      "clientMetadata" : {
        "driver" : {
          "name" : "PyMongo",
          "version" : "3.5.1"
        },
        "os" : {
          "type" : "Linux",
          "name" : "CentOS Linux 7.2.1511 Core",
          "architecture" : "x86_64",
          "version" : "3.10.0-327.22.2.el7.x86_64"
        },
        "platform" : "CPython 2.7.5.final.0"
      },
      "active" : true,
      "opid" : -1800224412,
      "secs_running" : 118,
      "microsecs_running" : NumberLong(118036322),
      "op" : "command",
      "ns" : "admin.$cmd",
      "query" : {
        "splitVector" : "mydb.mycoll",
        "keyPattern" : {
          "fd_shardkey" : "hashed"
        },
        "min" : {
          "fd_shardkey" : NumberLong("1585919574544073309")
        },
        "max" : {
          "fd_shardkey" : NumberLong("1585925300908251118")
        },
        "maxChunkSizeBytes" : NumberLong(67108864)
      },
      "numYields" : 422189,
      "locks" : {
        "Global" : "r",
        "Database" : "r",
        "Collection" : "r"
      },
      "waitingForLock" : false,
      "lockStats" : {
        "Global" : {
          "acquireCount" : {
            "r" : NumberLong(844380)
          }
        },
        "Database" : {
          "acquireCount" : {
            "r" : NumberLong(422190)
          },
          "acquireWaitCount" : {
            "r" : NumberLong(1)
          },
          "timeAcquiringMicros" : {
            "r" : NumberLong(94)
          }
        },
        "Collection" : {
          "acquireCount" : {
            "r" : NumberLong(422190)
          }
        }
      }
    },
    {
      "desc" : "conn734226",
      "threadId" : "139974009476864",
      "connectionId" : 734226,
      "client" : "xx.xx.xx.147:37786",
      "clientMetadata" : {
        "driver" : {
          "name" : "PyMongo",
          "version" : "3.5.1"
        },
        "os" : {
          "type" : "Linux",
          "name" : "CentOS Linux 7.2.1511 Core",
          "architecture" : "x86_64",
          "version" : "3.10.0-327.22.2.el7.x86_64"
        },
        "platform" : "CPython 2.7.5.final.0"
      },
      "active" : true,
      "opid" : -1800224471,
      "secs_running" : 118,
      "microsecs_running" : NumberLong(118048371),
      "op" : "command",
      "ns" : "admin.$cmd",
      "query" : {
        "splitVector" : "mydb.mycoll",
        "keyPattern" : {
          "fd_shardkey" : "hashed"
        },
        "min" : {
          "fd_shardkey" : NumberLong("1585919574544073309")
        },
        "max" : {
          "fd_shardkey" : NumberLong("1585925300908251118")
        },
        "maxChunkSizeBytes" : NumberLong(67108864)
      },
      "numYields" : 399251,
      "locks" : {
        "Global" : "r",
        "Database" : "r",
        "Collection" : "r"
      },
      "waitingForLock" : false,
      "lockStats" : {
        "Global" : {
          "acquireCount" : {
            "r" : NumberLong(798504)
          }
        },
        "Database" : {
          "acquireCount" : {
            "r" : NumberLong(399252)
          },
          "acquireWaitCount" : {
            "r" : NumberLong(1)
          },
          "timeAcquiringMicros" : {
            "r" : NumberLong(158)
          }
        },
        "Collection" : {
          "acquireCount" : {
            "r" : NumberLong(399252)
          }
        }
      }
    },
    {
      "desc" : "conn734224",
      "threadId" : "139976032712448",
      "connectionId" : 734224,
      "client" : "xx.xx.xx.147:37782",
      "clientMetadata" : {
        "driver" : {
          "name" : "PyMongo",
          "version" : "3.5.1"
        },
        "os" : {
          "type" : "Linux",
          "name" : "CentOS Linux 7.2.1511 Core",
          "architecture" : "x86_64",
          "version" : "3.10.0-327.22.2.el7.x86_64"
        },
        "platform" : "CPython 2.7.5.final.0"
      },
      "active" : true,
      "opid" : -1800226779,
      "secs_running" : 118,
      "microsecs_running" : NumberLong(118400690),
      "op" : "command",
      "ns" : "admin.$cmd",
      "query" : {
        "splitVector" : "mydb.mycoll",
        "keyPattern" : {
          "fd_shardkey" : "hashed"
        },
        "min" : {
          "fd_shardkey" : NumberLong("1585919574544073309")
        },
        "max" : {
          "fd_shardkey" : NumberLong("1585925300908251118")
        },
        "maxChunkSizeBytes" : NumberLong(67108864)
      },
      "numYields" : 416562,
      "locks" : {
        "Global" : "r",
        "Database" : "r",
        "Collection" : "r"
      },
      "waitingForLock" : false,
      "lockStats" : {
        "Global" : {
          "acquireCount" : {
            "r" : NumberLong(833126)
          }
        },
        "Database" : {
          "acquireCount" : {
            "r" : NumberLong(416563)
          },
          "acquireWaitCount" : {
            "r" : NumberLong(1)
          },
          "timeAcquiringMicros" : {
            "r" : NumberLong(200)
          }
        },
        "Collection" : {
          "acquireCount" : {
            "r" : NumberLong(416563)
          }
        }
      }
    },
    {
      "desc" : "conn734365",
      "threadId" : "139976806426368",
      "connectionId" : 734365,
      "client" : "xx.xx.xx.147:39492",
      "clientMetadata" : {
        "driver" : {
          "name" : "PyMongo",
          "version" : "3.5.1"
        },
        "os" : {
          "type" : "Linux",
          "name" : "CentOS Linux 7.2.1511 Core",
          "architecture" : "x86_64",
          "version" : "3.10.0-327.22.2.el7.x86_64"
        },
        "platform" : "CPython 2.7.5.final.0"
      },
      "active" : true,
      "opid" : -1800222592,
      "secs_running" : 117,
      "microsecs_running" : NumberLong(117677575),
      "op" : "command",
      "ns" : "admin.$cmd",
      "query" : {
        "splitVector" : "mydb.mycoll",
        "keyPattern" : {
          "fd_shardkey" : "hashed"
        },
        "min" : {
          "fd_shardkey" : NumberLong("1585919574544073309")
        },
        "max" : {
          "fd_shardkey" : NumberLong("1585925300908251118")
        },
        "maxChunkSizeBytes" : NumberLong(67108864)
      },
      "numYields" : 418779,
      "locks" : {
        "Global" : "r",
        "Database" : "r",
        "Collection" : "r"
      },
      "waitingForLock" : false,
      "lockStats" : {
        "Global" : {
          "acquireCount" : {
            "r" : NumberLong(837560)
          }
        },
        "Database" : {
          "acquireCount" : {
            "r" : NumberLong(418780)
          },
          "acquireWaitCount" : {
            "r" : NumberLong(1)
          },
          "timeAcquiringMicros" : {
            "r" : NumberLong(206)
          }
        },
        "Collection" : {
          "acquireCount" : {
            "r" : NumberLong(418780)
          }
        }
      }
    },
    {
      "desc" : "conn734227",
      "threadId" : "139976726423296",
      "connectionId" : 734227,
      "client" : "xx.xx.xx.147:37788",
      "clientMetadata" : {
        "driver" : {
          "name" : "PyMongo",
          "version" : "3.5.1"
        },
        "os" : {
          "type" : "Linux",
          "name" : "CentOS Linux 7.2.1511 Core",
          "architecture" : "x86_64",
          "version" : "3.10.0-327.22.2.el7.x86_64"
        },
        "platform" : "CPython 2.7.5.final.0"
      },
      "active" : true,
      "opid" : -1800224505,
      "secs_running" : 118,
      "microsecs_running" : NumberLong(118054608),
      "op" : "command",
      "ns" : "admin.$cmd",
      "query" : {
        "splitVector" : "mydb.mycoll",
        "keyPattern" : {
          "fd_shardkey" : "hashed"
        },
        "min" : {
          "fd_shardkey" : NumberLong("1585919574544073309")
        },
        "max" : {
          "fd_shardkey" : NumberLong("1585925300908251118")
        },
        "maxChunkSizeBytes" : NumberLong(67108864)
      },
      "numYields" : 390945,
      "locks" : {
        "Global" : "r",
        "Database" : "r",
        "Collection" : "r"
      },
      "waitingForLock" : false,
      "lockStats" : {
        "Global" : {
          "acquireCount" : {
            "r" : NumberLong(781892)
          }
        },
        "Database" : {
          "acquireCount" : {
            "r" : NumberLong(390946)
          },
          "acquireWaitCount" : {
            "r" : NumberLong(1)
          },
          "timeAcquiringMicros" : {
            "r" : NumberLong(198)
          }
        },
        "Collection" : {
          "acquireCount" : {
            "r" : NumberLong(390946)
          }
        }
      }
    }
  ],
  "ok" : 1
}

Details

Description

Attachments

Activity

People

Dates