<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:14:18 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-7355] mongoimport cannot import a dump in json array format larger than 16MB</title>
                <link>https://jira.mongodb.org/browse/SERVER-7355</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The following fails on 2.2.0. Seems to be related to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-6498?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&amp;amp;focusedCommentId=174809#comment-174809&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-6498?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&amp;amp;focusedCommentId=174809#comment-174809&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;C:\&amp;gt;mongodb\bin\mongoimport.exe --collection collection --file file.json&lt;br/&gt;
connected to: 127.0.0.1&lt;br/&gt;
Mon Oct 15 15:08:15 exception:read error, or input line too long (max length: 16&lt;br/&gt;
777216)&lt;br/&gt;
Mon Oct 15 15:08:15&lt;br/&gt;
Mon Oct 15 15:08:15 imported 0 objects&lt;br/&gt;
Mon Oct 15 15:08:15 ERROR: encountered 1 error&lt;/p&gt;</description>
                <environment></environment>
        <key id="53173">SERVER-7355</key>
            <summary>mongoimport cannot import a dump in json array format larger than 16MB</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="sverch">Shaun Verch</assignee>
                                    <reporter username="edgar88">edgar 88</reporter>
                        <labels>
                            <label>mongoimport</label>
                    </labels>
                <created>Mon, 15 Oct 2012 13:18:08 +0000</created>
                <updated>Mon, 11 Jul 2016 17:57:44 +0000</updated>
                            <resolved>Sat, 27 Apr 2013 01:19:08 +0000</resolved>
                                    <version>2.2.0</version>
                                    <fixVersion>2.5.0</fixVersion>
                                    <component>Tools</component>
                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="508109" author="bkiefer" created="Mon, 3 Mar 2014 17:14:18 +0000"  >&lt;p&gt;Shaun, sorry it&apos;s taken so long to get back to you. Here is the information you requested. One thing to note is that we are using 2.4.6-rc1 for our export, and do not have an easy way to upgrade those yet. I was using 2.6.0rc for the import.&lt;/p&gt;

&lt;p&gt;Here are the commands you requested.&lt;br/&gt;
mongoimport -h host --port port -u user -p pass -d database -c collection --file file.json&lt;br/&gt;
mongoexport -h host --port port -u user -p pass -d database -c collection --query somequery --out file.json&lt;/p&gt;

&lt;p&gt;The problem line turned out to be 18339991 bytes long. However, it was successfully exported using the command above.&lt;/p&gt;

&lt;p&gt;It definitely looks like the issue is related to the one you linked above.&lt;/p&gt;</comment>
                            <comment id="505064" author="sverch" created="Tue, 25 Feb 2014 19:01:42 +0000"  >&lt;p&gt;Hi Ben,&lt;/p&gt;

&lt;p&gt;Thanks for your report.  There are actually two related issues.  The original issue behind this ticket involved importing a collection that was exported using &quot;-&lt;del&gt;jsonArray&quot;.  Before this fix, the entire collection&apos;s JSON representation could not be larger than 16MB when using &quot;&lt;/del&gt;-jsonArray&quot;.&lt;/p&gt;

&lt;p&gt;However, some of the internal buffers are still 16MB, which means if individual documents are too large, we can still see this error.  See &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-12884&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-12884&lt;/a&gt; for more details.&lt;/p&gt;

&lt;p&gt;To confirm that this is actually your issue, could you provide the following:&lt;/p&gt;

&lt;p&gt;1. The command line flags you used with mongoexport and mongoimport.&lt;/p&gt;

&lt;p&gt;2. More details about the sizes of your documents.  For example, if you did not use &quot;--jsonArray&quot;, the size (in bytes) of each line of the file is the information that is relevant here.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
~Shaun Verch&lt;/p&gt;</comment>
                            <comment id="504845" author="bkiefer" created="Tue, 25 Feb 2014 15:31:28 +0000"  >&lt;p&gt;I&apos;m still getting this error on 2.6.0rc on windows.&lt;/p&gt;

&lt;p&gt;exception:read error, or input line too long (max length: 16777216)&lt;/p&gt;</comment>
                            <comment id="465362" author="mickdelaney" created="Wed, 4 Dec 2013 11:46:52 +0000"  >&lt;p&gt;i&apos;m getting this error on a json collection import to mongo running 2.5.4 on windows. &lt;/p&gt;

&lt;p&gt;call %MONGO_IMPORT% -h %SERVER% -d %DATABASE_NAME% -u %DATABASE_USER% -p %DATABASE_PWD% -c mycollection --drop --file mycollection.json&lt;/p&gt;

&lt;p&gt;connected to: localhost:27017&lt;br/&gt;
2013-12-04T11:44:38.054+0000 dropping: mycollection&lt;br/&gt;
exception:read error, or input line too long (max length: 16777216)&lt;br/&gt;
2013-12-04T11:44:38.169+0000 check 0 0&lt;br/&gt;
2013-12-04T11:44:38.170+0000 imported 0 objects&lt;br/&gt;
encountered 1 error(s)&lt;/p&gt;</comment>
                            <comment id="322747" author="auto" created="Sat, 27 Apr 2013 01:17:29 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;date&apos;: u&apos;2013-04-19T16:21:41Z&apos;, u&apos;name&apos;: u&apos;Shaun Verch&apos;, u&apos;email&apos;: u&apos;shaun.verch@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-7355&quot; title=&quot;mongoimport cannot import a dump in json array format larger than 16MB&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-7355&quot;&gt;&lt;del&gt;SERVER-7355&lt;/del&gt;&lt;/a&gt; (5/5) Added jstest for importing large JSON arrays&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/64a28e32e5e0584e12ddecc2b7a967b7b4d109ca&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/64a28e32e5e0584e12ddecc2b7a967b7b4d109ca&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="322746" author="auto" created="Sat, 27 Apr 2013 01:17:26 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;date&apos;: u&apos;2013-04-19T16:12:05Z&apos;, u&apos;name&apos;: u&apos;Shaun Verch&apos;, u&apos;email&apos;: u&apos;shaun.verch@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-7355&quot; title=&quot;mongoimport cannot import a dump in json array format larger than 16MB&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-7355&quot;&gt;&lt;del&gt;SERVER-7355&lt;/del&gt;&lt;/a&gt; (4/5) Buffering input stream to handle arbitrarily large single line JSON arrays&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/fbc18f3f1141373e75042769bc6249ee31c1f317&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/fbc18f3f1141373e75042769bc6249ee31c1f317&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="322745" author="auto" created="Sat, 27 Apr 2013 01:17:25 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;date&apos;: u&apos;2013-04-19T15:52:13Z&apos;, u&apos;name&apos;: u&apos;Shaun Verch&apos;, u&apos;email&apos;: u&apos;shaun.verch@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-7355&quot; title=&quot;mongoimport cannot import a dump in json array format larger than 16MB&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-7355&quot;&gt;&lt;del&gt;SERVER-7355&lt;/del&gt;&lt;/a&gt; (3/5) Remove inline JSON array conditionals from now separated branches&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/8376a849147d9a56cd46b71a4fd133e368e70d65&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/8376a849147d9a56cd46b71a4fd133e368e70d65&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="322744" author="auto" created="Sat, 27 Apr 2013 01:17:23 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;date&apos;: u&apos;2013-04-19T15:48:10Z&apos;, u&apos;name&apos;: u&apos;Shaun Verch&apos;, u&apos;email&apos;: u&apos;shaun.verch@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-7355&quot; title=&quot;mongoimport cannot import a dump in json array format larger than 16MB&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-7355&quot;&gt;&lt;del&gt;SERVER-7355&lt;/del&gt;&lt;/a&gt; (2/5) Copied main while loop to separate JSON array logic from one per line logic&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/2d38469a02bbe1d171d7ba789aefd94245af303c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/2d38469a02bbe1d171d7ba789aefd94245af303c&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="322743" author="auto" created="Sat, 27 Apr 2013 01:17:20 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;date&apos;: u&apos;2013-04-17T20:42:27Z&apos;, u&apos;name&apos;: u&apos;Shaun Verch&apos;, u&apos;email&apos;: u&apos;shaun.verch@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-7355&quot; title=&quot;mongoimport cannot import a dump in json array format larger than 16MB&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-7355&quot;&gt;&lt;del&gt;SERVER-7355&lt;/del&gt;&lt;/a&gt; (1/5) Factored out insert function in mongoimport&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/12dd5bcedc0a36bb82a8093ed1bc8f036160173f&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/12dd5bcedc0a36bb82a8093ed1bc8f036160173f&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="175874" author="edgar88" created="Wed, 17 Oct 2012 05:56:20 +0000"  >&lt;p&gt;Hi Ben,&lt;/p&gt;

&lt;p&gt;The invididual JSON-objects in the database are not exceeding 16 mb, they are twitter messages captured from the twitter stream. I think the problem lies in that the objects are not line delimited (perhaps because I used the option --jsonarray). The first 10.000 characters of the file (all on the first line) are pasted here: &lt;a href=&quot;http://pastebin.com/ehVZjcnD&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://pastebin.com/ehVZjcnD&lt;/a&gt; Note &lt;/p&gt;</comment>
                            <comment id="175824" author="benjamin.becker" created="Wed, 17 Oct 2012 00:30:27 +0000"  >&lt;p&gt;Hi Edgar,&lt;/p&gt;

&lt;p&gt;It sounds like mongoexport with --jsonarray produced a line longer than 16mb.  There are cases where the JSON format can be larger than BSON, which we probably need to handle in mongoexport.  That said, in generally these tools are only used for importing/exporting data on MongoDB, while mongodump and mongorestore will preserve an exact copy of the data between the dump and restore process.&lt;/p&gt;

&lt;p&gt;This seems to be working as expected, but would it be possible to proved the long line that produces this error to confirm?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ben&lt;/p&gt;</comment>
                            <comment id="175293" author="edgar88" created="Tue, 16 Oct 2012 07:18:05 +0000"  >&lt;p&gt;importing with the option --jsonArray gives:&lt;/p&gt;

&lt;p&gt;Tue Oct 16 09:17:19 exception:JSONArray file too large&lt;br/&gt;
Tue Oct 16 09:17:19 warning: log line attempted (16384k) over max size(10k), pri&lt;br/&gt;
nting beginning and end ... &lt;/p&gt;</comment>
                            <comment id="175284" author="edgar88" created="Tue, 16 Oct 2012 07:05:38 +0000"  >&lt;p&gt;When I try to read the file created by mongoexport via a BufferedReader in Java, I get an out of memory error. It seems to me that the JSON file is not line delimited and therefore the error &quot;input line too long&quot; is thrown by mongodb&lt;/p&gt;</comment>
                            <comment id="174818" author="edgar88" created="Mon, 15 Oct 2012 13:36:49 +0000"  >&lt;p&gt;The collection file.json was created just before the import, using mongoexport --collection collection --file file.json --jsonarray.&lt;br/&gt;
It is a ~ 2 GB database.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="114496">TOOLS-63</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="103307">TOOLS-73</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 17 Oct 2012 00:30:27 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        9 years, 50 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            9 years, 50 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10022"><![CDATA[Windows]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>auto</customfieldvalue>
            <customfieldvalue>benjamin.becker</customfieldvalue>
            <customfieldvalue>bkiefer</customfieldvalue>
            <customfieldvalue>edgar88</customfieldvalue>
            <customfieldvalue>mickdelaney</customfieldvalue>
            <customfieldvalue>sverch</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrnl87:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrjznz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>30326</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10166" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Tests Written</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10154"><![CDATA[Complete]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hs7ba7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>