[SERVER-424] mongoimport mangles large numbers (BIGINT) on import via CSV. Created: 13/Nov/09 Updated: 12/Jul/16 Resolved: 13/Nov/09 |
|
| Status: | Closed |
| Project: | Core Server |
| Component/s: | Usability |
| Affects Version/s: | 1.1.3 |
| Fix Version/s: | 1.1.4 |
| Type: | Bug | Priority: | Minor - P4 |
| Reporter: | Alexey Zilber | Assignee: | Eliot Horowitz (Inactive) |
| Resolution: | Done | Votes: | 0 |
| Labels: | None | ||
| Remaining Estimate: | Not Specified | ||
| Time Spent: | Not Specified | ||
| Original Estimate: | Not Specified | ||
| Environment: |
Linux, AWS m1.large instance, CentOS 5.2 x86_64 |
||
| Participants: |
| Description |
|
This is a show stopper folks. Exported data from mysql in csv format. Imported into MongoDB. Some of the mysql BIGINT fields got mangled and created dupes, this relates to the dupe seen in ( From the CSV: [root@ip-10-244-51-31 mnt]# grep -i '501272713' Farm.csv s:9:\"harvested\";i:0;s:5:\"units\";i:31;}i:1;a:11:{s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;s:5:\"plant\";s:1:\"1\";s:5:\"stage\";i:2;s:10:\"stage_next\";i:3;s:11:\"stage_start\";i:1257862130;s:6:\"stolen\";i:0;s:9:\"stolen_by\";a:0:{}s:9:\"harvested\";i:0;}}i:1;a:2:{i:0;a:12:{s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;s:5:\"plant\";s:1:\"5\";s:5:\"stage\";i:2;s:10:\"stage_next\";i:3;s:11:\"stage_start\";i:1257862130;s:6:\"stolen\";i:0;s:9:\"stolen_by\";a:0:{}s:9:\"harvested\";i:0;s:8:\"weeds_by\";b:0;}i:1;a:4:{s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;}}i:2;a:2:{i:0;a:12:{s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;s:5:\"plant\";s:1:\"4\";s:5:\"stage\";i:2;s:10:\"stage_next\";i:3;s:11:\"stage_start\";i:1257862130;s:6:\"stolen\";i:0;s:9:\"stolen_by\";a:0:{}s:9:\"harvested\";i:0;s:7:\"bugs_by\";b:0;}i:1;a:4: {s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;}}}","","a:4:{i:0;a:3: {s:3:\"uid\";s:10:\"1542862066\";s:1:\"t\";i:1257862132;s:6:\"action\";s:38:\"volé 2 Aubergines de Ferme de Claire!\";}i:1;a:3: {s:3:\"uid\";s:10:\"1542862066\";s:1:\"t\";i:1257862132;s:6:\"action\";s:44:\"a éliminé vos insectes le Ferme de Claire!\";}i:2;a:3: {s:3:\"uid\";s:10:\"1542862066\";s:1:\"t\";i:1257862132;s:6:\"action\";s:49:\"a éliminé vos mauves herbes le Ferme de Claire!\";}i:3;a:3:{s:3:\"uid\";s:10:\"1542862066\";s:1:\"t\";i:1257862131;s:6:\"action\";s:41:\"a arrosé vos plantes le Ferme de Claire!\";}}" s:9:\"harvested\";i:0;s:5:\"units\";i:31;}i:1;a:12:{s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;s:5:\"plant\";s:1:\"1\";s:5:\"stage\";i:5;s:10:\"stage_next\";i:6;s:11:\"stage_start\";i:1255195152;s:6:\"stolen\";i:4;s:9:\"stolen_by\";a:1: {i:1494751562;i:4;}s:9:\"harvested\";i:0;s:5:\"units\";i:22;}}i:1;a:2:{i:0;a:13:{s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;s:5:\"plant\";s:1:\"5\";s:5:\"stage\";i:5;s:10:\"stage_next\";i:6;s:11:\"stage_start\";i:1255216752;s:6:\"stolen\";i:1;s:9:\"stolen_by\";a:1: {i:1494751562;i:1;}s:9:\"harvested\";i:0;s:8:\"weeds_by\";b:0;s:5:\"units\";i:24;}i:1;a:4:{s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;}}i:2;a:2:{i:0;a:13:{s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;s:5:\"plant\";s:1:\"4\";s:5:\"stage\";i:5;s:10:\"stage_next\";i:6;s:11:\"stage_start\";i:1255205952;s:6:\"stolen\";i:2;s:9:\"stolen_by\";a:1: {i:1494751562;i:2;}s:9:\"harvested\";i:0;s:7:\"bugs_by\";b:0;s:5:\"units\";i:19;}i:1;a:4: {s:3:\"dry\";i:0;s:5:\"weeds\";i:0;s:4:\"bugs\";i:0;s:10:\"fertilized\";i:0;}}}","","a:4:{i:0;a:3: {s:3:\"uid\";s:10:\"1494751562\";s:1:\"t\";i:1255220951;s:6:\"action\";s:31:\"volé 2 Carottes de army ferme!\";}i:1;a:3: {s:3:\"uid\";s:10:\"1494751562\";s:1:\"t\";i:1255220950;s:6:\"action\";s:30:\"volé 1 Patates de army ferme!\";}i:2;a:3: {s:3:\"uid\";s:10:\"1494751562\";s:1:\"t\";i:1255220949;s:6:\"action\";s:33:\"volé 4 Aubergines de army ferme!\";}i:3;a:3:{s:3:\"uid\";s:10:\"1494751562\";s:1:\"t\";i:1255220947;s:6:\"action\";s:28:\"volé 4 Radis de army ferme!\";}}" From MongoDB: > var cursor = db.Farm.find( { uid:501272713 }) { s:9:\\"harvested\\";i:0;s:5:\\"units\\";i:31;}i:1;a:11:{s:3:\\"dry\\";i:0;s:5:\\"weeds\\";i:0;s:4:\\"bugs\\";i:0;s:10:\\"fertilized\\";i:0;s:5:\\"plant\\";s:1:\\"1\\";s:5:\\"stage\\";i:2;s:10:\\"stage_next\\";i:3;s:11:\\"stage_start\\";i:1257862130;s:6:\\"stolen\\";i:0;s:9:\\"stolen_by\\";a:0:{}s:9:\\"harvested\\";i:0;}}i:1;a:2:{i:0;a:12:{s:3:\\"dry\\";i:0;s:5:\\"weeds\\";i:0;s:4:\\"bugs\\";i:0;s:10:\\"fertilized\\";i:0;s:5:\\"plant\\";s:1:\\"5\\";s:5:\\"stage\\";i:2;s:10:\\"stage_next\\";i:3;s:11:\\"stage_start\\";i:1257862130;s:6:\\"stolen\\";i:0;s:9:\\"stolen_by\\";a:0:{}s:9:\\"harvested\\";i:0;s:8:\\"weeds_by\\";b:0;}i:1;a:4:{s:3:\\"dry\\";i:0;s:5:\\"weeds\\";i:0;s:4:\\"bugs\\";i:0;s:10:\\"fertilized\\";i:0;}}i:2;a:2:{i:0;a:12:{s:3:\\"dry\\";i:0;s:5:\\"weeds\\";i:0;s:4:\\"bugs\\";i:0;s:10:\\"fertilized\\";i:0;s:5:\\"plant\\";s:1:\\"4\\";s:5:\\"stage\\";i:2;s:10:\\"stage_next\\";i:3;s:11:\\"stage_start\\";i:1257862130;s:6:\\"stolen\\";i:0;s:9:\\"stolen_by\\";a:0:{}s:9:\\"harvested\\";i:0;s:7:\\"bugs_by }}\"", i:1;a:3: {s:3:\\"uid\\";s:10:\\"1542862066\\";s:1:\\"t\\";i:1257862132;s:6:\\"action\\";s:44:\\"a éliminé vos insectes le Ferme de Claire!\\";}i:2;a:3: {s:3:\\"uid\\";s:10:\\"1542862066\\";s:1:\\"t\\";i:1257862132;s:6:\\"action\\";s:49:\\"a éliminé vos mauves herbes le Ferme de Claire!\\";}i:3;a:3:{s:3:\\"uid\\";s:10:\\"1542862066\\";s:1:\\"t\\";i:1257862131;s:6:\\"action\\";s:41: s:9:\\"harvested\\";i:0;s:5:\\"units\\";i:31;}i:1;a:12:{s:3:\\"dry\\";i:0;s:5:\\"weeds\\";i:0;s:4:\\"bugs\\";i:0;s:10:\\"fertilized\\";i:0;s:5:\\"plant\\";s:1:\\"1\\";s:5:\\"stage\\";i:5;s:10:\\"stage_next\\";i:6;s:11:\\"stage_start\\";i:1255195152;s:6:\\"stolen\\";i:4;s:9:\\"stolen_by s:9:\\"harvested\\";i:0;s:5:\\"units\\";i:22;}}i:1;a:2:{i:0;a:13:{s:3:\\"dry\\";i:0;s:5:\\"weeds\\";i:0;s:4:\\"bugs\\";i:0;s:10:\\"fertilized\\";i:0;s:5:\\"plant\\";s:1:\\"5\\";s:5:\\"stage\\";i:5;s:10:\\"stage_next\\";i:6;s:11:\\"stage_start\\";i:1255216752;s:6:\\"stolen\\";i:1;s:9:\\"stolen_by s:9:\\"harvested\\";i:0;s:8:\\"weeds_by\\";b:0;s:5:\\"units\\";i:24;}i:1;a:4:{s:3:\\"dry\\";i:0;s:5:\\"weeds\\";i:0;s:4:\\"bugs\\";i:0;s:10:\\"fertilized\\";i:0;}}i:2;a:2:{i:0;a:13:{s:3:\\"dry\\";i:0;s:5:\\"weeds\\";i:0;s:4:\\"bugs\\";i:0;s:10:\\"fertilized\\";i:0;s:5:\\"plant\\";s:1:\\"4\\";s:5:\\"stage\\";i:5;s:10:\\"stage_next\\";i:6;s:11:\\"stage_start\\";i:1255205952;s:6:\\"stolen\\";i:2;s:9:\\"stolen_by s:9:\\"harvested\\";i:0;s:7:\\"bugs_by\\";b:0;s:5:\\"units }}\"", i:1;a:3: {s:3:\\"uid\\";s:10:\\"1494751562\\";s:1:\\"t\\";i:1255220950;s:6:\\"action\\";s:30:\\"volé 1 Patates de army ferme!\\";}i:2;a:3: {s:3:\\"uid\\";s:10:\\"1494751562\\";s:1:\\"t\\";i:1255220949;s:6:\\"action\\";s:33:\\"volé 4 Aubergines de army ferme!\\";}i:3;a:3:{s:3:\\"uid\\";s:10:\\"1494751562\\";s:1:\\"t\\";i:1255220947;s:6:\\"action\\";s:28: O M G!!!! |
| Comments |
| Comment by Eliot Horowitz (Inactive) [ 13/Nov/09 ] | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Please put the version using in the environment. | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| Comment by auto [ 13/Nov/09 ] | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Author: {'name': 'Eliot Horowitz', 'email': 'eliot@10gen.com'}Message: fix import with long long | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| Comment by Alexey Zilber [ 13/Nov/09 ] | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
I've done a followup test using quoted and unquoted csv data. I can replicate this without a problem. FARMUID db has everything encapsulated in quotes, and FARMUID2 is set to optional, so it doesn't enclose int. Here are the results: > use FARMUID > var cursor = db.Farm.find( { uid:"\"501272713\"" }) { "_id" : ObjectId("4afdf5dc133aac262779d052"), "uid" : "\"501272713\"", "name" : "\"\"", "last_change" : "\"2009-11-10 08:08:52\"" } > var cursor = db.Farm.find({ last_change:"\"2009-10-10 19:29:11\"" }) > while (cursor.hasNext()) { print(tojson(cursor.next())); } { > use FARMUID2 > var cursor = db.Farm.find( { uid:501272713 }) { Please fix! | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| Comment by Alexey Zilber [ 13/Nov/09 ] | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Fyi, this is the mysql schema: mysql> describe Farm;
------------
------------ |