[SERVER-5872] autosplit would be triggered before data size touch chunksize. The return message "updatedExisting" of update is missed Created: 18/May/12 Updated: 11/Jul/16 Resolved: 15/Jun/12 |
|
| Status: | Closed |
| Project: | Core Server |
| Component/s: | Sharding |
| Affects Version/s: | 2.0.2 |
| Fix Version/s: | 2.0.7, 2.1.2 |
| Type: | Bug | Priority: | Minor - P4 |
| Reporter: | eason lin | Assignee: | siddharth.singh@10gen.com |
| Resolution: | Done | Votes: | 0 |
| Labels: | None | ||
| Remaining Estimate: | Not Specified | ||
| Time Spent: | Not Specified | ||
| Original Estimate: | Not Specified | ||
| Environment: |
singleShard mongos> db.settings.find() { "_id" : "chunksize", "value" : 64 } |
||
| Operating System: | Linux |
| Participants: |
| Description |
|
chunksize was 64, but autosplit would be triggered before data size touch it. ===Python Test Script===
===Test Script Output=== , u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018': {u'ok': 1.0, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a8f'), u'err': None, u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352771L}}} mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a91'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352773L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a92'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352774L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a93'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352775L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a94'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352776L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a95'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352777L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a96'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352778L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a97'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352779L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a98'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352780L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a99'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352781L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a9a'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352782L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a9b'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352783L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a9c'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352784L}mesg={u'shardRawGLE': {u'P1.trend.lava.tw:27019,P2.trend.lava.tw:27019,P3.trend.lava.tw:27019': {u'connectionId': 5705, u'ok': 1.0, u'err': None, u'n': 0}, u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018': {u'connectionId': 48862, u'lastOp': 5743692655800352785L, u'ok': 1.0, u'err': None, u'n': 0}}, u'ok': 1.0, u'shards': [u'P1.trend.lava.tw:27019,P2.trend.lava.tw:27019,P3.trend.lava.tw:27019', u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018'], u'err': None, u'n': 0} ===The changelog after test=== , "shardkey1" : { $minKey : 1 }}, "max" : { "shardkey2" : { $maxKey : 1 }, "shardkey1" : { $maxKey : 1 }}, "lastmod" : { "t" : 1000, "i" : 0 }}, "left" : { "min" : { "shardkey2" : { $minKey : 1 }, "shardkey1" : { $minKey : 1 }}, "max" : { "shardkey2" : "test1", "shardkey1" : "test1" }, "lastmod" : { "t" : 1000, "i" : 1 }}, "right" : { "min" : { "shardkey2" : "test1", "shardkey1" : "test1" }, "max" : { "shardkey2" : { $maxKey : 1 }, "shardkey1" : { $maxKey : 1 }}, "lastmod" : { "t" : 1000, "i" : 2 }} } } ===Log of mongos.log after test=== } Fri May 18 03:02:38 [conn8299] created new distributed lock for testsplit.coll_5 on P1.trend.lava.tw:27019,P2.trend.lava.tw:27019,P3.trend.lava.tw:27019 ( lock timeout : 900000, ping interval : 30000, process : 0 ) max: { shardkey2: MaxKey, shardkey1: MaxKey }on: { shardkey2: "test1", shardkey1: "test1" }(splitThreshold 921) Fri May 18 03:02:38 [conn8299] end connection 172.16.2.60:50953 |
| Comments |
| Comment by auto [ 09/Jul/12 ] |
|
Author: {u'date': u'2012-07-09T10:59:09-07:00', u'name': u'Siddharth Singh', u'email': u'singhsiddharth@gmail.com'}Message: When autosplit, 'updatedExisting': False is missing. |
| Comment by auto [ 03/Jul/12 ] |
|
Author: {u'date': u'2012-07-03T08:42:18-07:00', u'email': u'milkie@10gen.com', u'name': u'Eric Milkie'}Message: |
| Comment by auto [ 15/Jun/12 ] |
|
Author: {u'date': u'2012-06-15T08:47:39-07:00', u'email': u'singhsiddharth@gmail.com', u'name': u'Siddharth Singh'}Message: Use internalScopedDBConnection when we talk to other shards during |
| Comment by siddharth.singh@10gen.com [ 15/Jun/12 ] |
|
Found the problem source. During autosplits, mongos talks to the shards. The existing code, however, was using the same connection (the one over which we were doing upserts) to communicate with the shards. getLastError command in those cases was reporting the information from the last database operation which was mongos querying the shard and hence the updatedExisting message was missing. Will push the fix to the master branch soon. Thanks for reporting this to us. |
| Comment by eason lin [ 14/Jun/12 ] |
|
Thank you for your update. What I meant about 'return message of updatedExisting is missed' is the first 15 times insertion were fine but there was no updatedExisting field at 16 times insertion. I copyed the Test Script Output and heightlight it to red color. (Execuse me I couldn't find way to edit the description, so I had to copy it.) ===Test Script Output=== , u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018': {u'ok': 1.0, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a8f'), u'err': None, u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352771L}}} mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a91'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352773L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a92'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352774L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a93'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352775L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a94'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352776L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a95'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352777L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a96'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352778L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a97'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352779L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a98'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352780L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a99'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352781L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a9a'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352782L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a9b'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352783L}mesg= {u'ok': 1.0, u'err': None, u'upserted': ObjectId('4fb5b20ab7487b8bf2693a9c'), u'singleShard': u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018', u'connectionId': 48862, u'n': 1, u'updatedExisting': False, u'lastOp': 5743692655800352784L}mesg={u'shardRawGLE': {u'P1.trend.lava.tw:27019,P2.trend.lava.tw:27019,P3.trend.lava.tw:27019': {u'connectionId': 5705, u'ok': 1.0, u'err': None, u'n': 0}, u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018': {u'connectionId': 48862, u'lastOp': 5743692655800352785L, u'ok': 1.0, u'err': None, u'n': 0}}, u'ok': 1.0, u'shards': [u'P1.trend.lava.tw:27019,P2.trend.lava.tw:27019,P3.trend.lava.tw:27019', u'P1.trend.lava.tw_P2.trend.lava.tw_P3.trend.lava.tw/P1.trend.lava.tw:27018,P3.trend.lava.tw:27018,P2.trend.lava.tw:27018'], u'err': None, u'n': 0} |
| Comment by siddharth.singh@10gen.com [ 13/Jun/12 ] |
|
It is normal for the autosplits to be triggered before the data size touches the chunksize. updatedExisting field has a boolean value and in the logs that you have posted it does show up as 'updatedExisting': False so I am not sure what did you mean by 'return message of updatedExisting is missed' ? |