Details
-
Question
-
Resolution: Done
-
Critical - P2
-
None
-
2.6.1
-
None
Description
I am running a MR on mongodb, It silently drops records when I try to do a denormalize operation on 10000 records, it happens somewhere in the middle of the collections (~5K). I tried removing near by records (when sorted), its nothing wrong with the data.
Here is my code:
mapOrd = function() {
|
var values = {
|
customerId: this.customerId,
|
orderNr: this.orderNr,
|
productId: this.productId
|
};
|
emit(this.productId, values);
|
};
|
|
|
mapPrd = function() {
|
var values = {
|
code: this.code
|
};
|
emit(this.id, values);
|
};
|
|
|
reduceOrdPrd = function(k, values) {
|
var result = {};
|
values.forEach(function(value) {
|
var field;
|
if ("orderNr" in value) {
|
if (!("cust_ids" in result)) {
|
result.cust_ids = [];
|
}
|
result.cust_ids.push(value);
|
} else {
|
for (field in value) {
|
if (value.hasOwnProperty(field) ) {
|
result[field] = value[field];
|
}
|
};
|
}
|
});
|
return result;
|
};
|
db.prd_ord.drop();
|
|
|
db.order_10000.mapReduce(mapOrd, reduceOrdPrd, {"out": {"reduce": "prd_ord"}, "sort": {"productId": 1}});
|
I am running this on a machine which has very low config. 512 MB RAM with 1GB of SWAP memory.
But whatever the case, it should not silently (randomly) drop elements of groups.
The objects are not exceeding the BSON object limit.(just 10 to 15 array of objects per key).
Any suggestions, what could be causing this issue?
My code is working as designed. The only issue is dropping records for same emit key at a particular point after 5500 records in my code. e.g. productid=553 (emit key) has 12 elements.which is ending at 5502 nd record in input collection order_10000. 5501st and 5502nd record is getting dropped from the output of MR.
Input Order_10000:
db.order_10000.find({productId:553}).pretty();
|
{
|
"_id" : ObjectId("53f75e2ab4e41522bccf3410"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000058)
|
}
|
{
|
"_id" : ObjectId("53f75e2ab4e41522bccf3411"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000059)
|
}
|
{
|
"_id" : ObjectId("53f75e2ab4e41522bccf3412"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000060)
|
}
|
{
|
"_id" : ObjectId("53f75e2ab4e41522bccf3413"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000061)
|
}
|
{
|
"_id" : ObjectId("53f75e2ab4e41522bccf3414"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000062)
|
}
|
{
|
"_id" : ObjectId("53f75e2eb4e41522bccf3415"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderLineNr" : NumberLong(1),
|
"orderNr" : NumberLong(6000063)
|
}
|
{
|
"_id" : ObjectId("53f75e2eb4e41522bccf3416"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000064)
|
}
|
{
|
"_id" : ObjectId("53f75e2eb4e41522bccf3417"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000065)
|
}
|
{
|
"_id" : ObjectId("53f75e2eb4e41522bccf3418"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000066)
|
}
|
{
|
"_id" : ObjectId("53f75e97b4e41522bccf3419"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000067)
|
}
|
{
|
"_id" : ObjectId("53f75e97b4e41522bccf341a"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000068)
|
}
|
{
|
"_id" : ObjectId("53f76127b4e41522bccf341e"),
|
"customerId" : NumberLong(5699),
|
"productId" : NumberLong(553),
|
"orderNr" : NumberLong(6000071)
|
}
|
|
|
Output of MR:
db.prd_ord.find({'value.cust_ids.productId': 553}).pretty();
|
{
|
"_id" : NumberLong(553),
|
"value" : {
|
"cust_ids" : [
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000068),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000067),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000066),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000065),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000064),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000063),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000062),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000061),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000060),
|
"productId" : NumberLong(553)
|
},
|
{
|
"customerId" : NumberLong(5699),
|
"orderNr" : NumberLong(6000059),
|
"productId" : NumberLong(553)
|
}
|
],
|
"cust_ids_length" : 10
|
}
|
}
|
|
|
Next group starts fine from here. Same thing happens if a particular group crosses next threshold.
Am I missing some configuration or its a known bug