Details
Description
A query with both sort and limit, and which does not have an index to provide the sort, can fail on mongos even though it succeeds if you run it directly on each of the shards. Use the steps below to reproduce.
// Use mtools to setup the cluster and populate it with data.
|
$ mlaunch --sharded 2 --single -v
|
$ mgenerate '{"foo": {"$string": {"length": 10000}}, "bar": "$number", "baz": 0}' -n 20000
|
|
// Connect to mongos and shard the collection. Disable the balancer so that the data remains imbalanced,
|
// as there needs to be enough data on one of the shards to exceed the in-memory sort limit.
|
$ mongo
|
MongoDB shell version: 2.8.0-rc2
|
connecting to: test
|
mongos> db.version()
|
2.8.0-rc2
|
mongos> db.adminCommand({enableSharding: "test"})
|
{ "ok" : 1 }
|
mongos> db.adminCommand({shardCollection: "test.mgendata", key: {_id: 1}})
|
{ "collectionsharded" : "test.mgendata", "ok" : 1 }
|
mongos> sh.disableBalancing("test.mgendata")
|
|
// Connect to the shard with most of the data. The query succeeds on the shard until the limit
|
// becomes 3337 or higher. Then it fails due to hitting a memory limit. This is expected behavior.
|
$ mongo --port 27018
|
MongoDB shell version: 2.8.0-rc2
|
connecting to: 127.0.0.1:27018/test
|
> db.mgendata.count()
|
17950
|
> db.mgendata.find().sort({bar: 1}).limit(2000).itcount()
|
2000
|
> db.mgendata.find().sort({bar: 1}).limit(3336).itcount()
|
3336
|
> db.mgendata.find().sort({bar: 1}).limit(3337).itcount()
|
2014-12-10T20:39:17.781-0500 I QUERY Error: error: {
|
"$err" : "Executor error: Overflow sort stage buffered data usage of 33563546 bytes exceeds internal limit of 33554432 bytes",
|
"code" : 17144
|
}
|
at Error (<anonymous>)
|
at DBQuery.next (src/mongo/shell/query.js:259:15)
|
at DBQuery.itcount (src/mongo/shell/query.js:352:14)
|
at (shell):1:47 at src/mongo/shell/query.js:259
|
|
// Connect to mongos and do the same thing. This time we hit the memory limit even
|
// if the limit is small, which is the bug!
|
mongos> db.mgendata.find().sort({bar: 1}).limit(500).itcount()
|
2014-12-10T20:41:55.832-0500 I QUERY Error: error: {
|
"$err" : "getMore executor error: Overflow sort stage buffered data usage of 33563546 bytes exceeds internal limit of 33554432 bytes",
|
"code" : 17406
|
}
|
at Error (<anonymous>)
|
at DBQuery.next (src/mongo/shell/query.js:259:15)
|
at DBQuery.itcount (src/mongo/shell/query.js:352:14)
|
at (shell):1:46 at src/mongo/shell/query.js:259
|
Original repro steps
st = new ShardingTest({ shards: 2, chunkSize: 1, other: { separateConfig: true, nopreallocj: 1 }});
|
var db = st.s.getDB('test');
|
var mongosCol = db.getCollection('skip');
|
db.adminCommand({ enableSharding: 'test' });
|
db.adminCommand({ shardCollection: 'test.skip', key: { x: 1 }});
|
|
var i = 0;
|
var filler = new Array(1024).toString();
|
// create enough data to exceed 32MB limit
|
while (i < 32*1024 ) {
|
var bulk = [];
|
for (j = 0; j < 1024; j++) {
|
bulk.push({x:i+j, y:i+j, z:filler});
|
}
|
mongosCol.insert(bulk);
|
i += j;
|
}
|
jsTest.log(mongosCol.count() + " documents in " + mongosCol);
|
|
|
// test that direct connect to shard returns a document just below in-mem sort limit
|
var shardCol = st.shard0.getDB('test').getCollection('skip');
|
jsTest.log('test query succeeds directly on shard with skip 30000');
|
assert.eq(1, shardCol.find().sort({y:1}).skip(30000).limit(1).itcount());
|
|
// test that exceeding the in-memory sort limit errors on mongod
|
jsTest.log('test that there's an error on mongos with skip 32000');
|
assert.throws( function(){ mongosCol.find().sort({y:1}).skip(32000).limit(1).itcount() });
|
|
// test that exceeding the in-memory sort limit errors on mongos
|
jsTest.log('test that there's an error on mongos with skip 32000');
|
assert.throws( function(){ mongosCol.find().sort({y:1}).skip(32000).limit(1).itcount() });
|
|
// test that below limit should succeed on mongos
|
jsTest.log('test query succeeds on mongos with skip 30000');
|
assert.eq(1, mongosCol.find().sort({y:1}).skip(30000).limit(1).itcount());
|
Attachments
Issue Links
- is related to
-
SERVER-14299 For sharded limit=N queries with sort, mongos can request >N results from shard
-
- Closed
-