<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:43:42 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-17225] [RocksDB] Store all data in a single column family</title>
                <link>https://jira.mongodb.org/browse/SERVER-17225</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Currently, we store collections and indexes in separate column families. This is no-go for Parse workloads, since they have a lot of collections and we can&apos;t scale column families that much.&lt;/p&gt;

&lt;p&gt;The idea is to add a 4-byte prefix to every key. Each collection and index will get their own prefix.&lt;/p&gt;</description>
                <environment></environment>
        <key id="183230">SERVER-17225</key>
            <summary>[RocksDB] Store all data in a single column family</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="benety.goh@mongodb.com">Benety Goh</assignee>
                                    <reporter username="igor">Igor Canadi</reporter>
                        <labels>
                    </labels>
                <created>Mon, 9 Feb 2015 02:00:19 +0000</created>
                <updated>Wed, 25 Jan 2017 22:01:24 +0000</updated>
                            <resolved>Tue, 10 Feb 2015 21:59:44 +0000</resolved>
                                                    <fixVersion>3.0.0-rc9</fixVersion>
                    <fixVersion>3.1.0</fixVersion>
                                    <component>Storage</component>
                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="832774" author="ramon.fernandez" created="Thu, 19 Feb 2015 16:50:07 +0000"  >&lt;p&gt;Notes on v3.0 backports: &lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;&lt;a href=&quot;https://github.com/mongodb/mongo/commit/eaed5ed84ced12f738264ca4645733a5c1db420b&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;eaed5ed8&lt;/a&gt; includes the backport to v3.0 for &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17223&quot; title=&quot;Disable updateWithDamages support for RocksDB&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17223&quot;&gt;&lt;del&gt;SERVER-17223&lt;/del&gt;&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17225&quot; title=&quot;[RocksDB] Store all data in a single column family&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17225&quot;&gt;&lt;del&gt;SERVER-17225&lt;/del&gt;&lt;/a&gt; (this ticket).&lt;/li&gt;
	&lt;li&gt;if &lt;a href=&quot;https://github.com/mongodb/mongo/commit/aa9980b&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;aa9980b&lt;/a&gt; makes it into the v3.0 branch later on, we&apos;ll need to revert &lt;a href=&quot;https://github.com/mongodb/mongo/commit/eaed5ed84ced12f738264ca4645733a5c1db420b&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;eaed5ed8&lt;/a&gt; and apply both &lt;a href=&quot;https://github.com/mongodb/mongo/commit/c9e43db56a8f1a48c50097060aa9a8edc3621efc&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;c9e43db5&lt;/a&gt; (&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17223&quot; title=&quot;Disable updateWithDamages support for RocksDB&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17223&quot;&gt;&lt;del&gt;SERVER-17223&lt;/del&gt;&lt;/a&gt;) and &lt;a href=&quot;https://github.com/mongodb/mongo/commit/a31cc114a89a3645e97645805ba77db32c433dce&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;a31cc114&lt;/a&gt; (&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17225&quot; title=&quot;[RocksDB] Store all data in a single column family&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17225&quot;&gt;&lt;del&gt;SERVER-17225&lt;/del&gt;&lt;/a&gt;).&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="832762" author="xgen-internal-githook" created="Thu, 19 Feb 2015 16:37:30 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;igorcanadi&apos;, u&apos;name&apos;: u&apos;Igor Canadi&apos;, u&apos;email&apos;: u&apos;icanadi@fb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17225&quot; title=&quot;[RocksDB] Store all data in a single column family&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17225&quot;&gt;&lt;del&gt;SERVER-17225&lt;/del&gt;&lt;/a&gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;RocksDB&amp;#93;&lt;/span&gt; Store all data in a single column family &lt;span class=&quot;error&quot;&gt;&amp;#91;for v3.0&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Signed-off-by: Ramon Fernandez &amp;lt;ramon.fernandez@mongodb.com&amp;gt;&lt;br/&gt;
Branch: v3.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/eaed5ed84ced12f738264ca4645733a5c1db420b&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/eaed5ed84ced12f738264ca4645733a5c1db420b&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="826295" author="xgen-internal-githook" created="Tue, 10 Feb 2015 21:59:13 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;igorcanadi&apos;, u&apos;name&apos;: u&apos;Igor Canadi&apos;, u&apos;email&apos;: u&apos;icanadi@fb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17225&quot; title=&quot;[RocksDB] Store all data in a single column family&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17225&quot;&gt;&lt;del&gt;SERVER-17225&lt;/del&gt;&lt;/a&gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;RocksDB&amp;#93;&lt;/span&gt; Store all data in a single column family&lt;/p&gt;

&lt;p&gt;Signed-off-by: Benety Goh &amp;lt;benety@mongodb.com&amp;gt;&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/a31cc114a89a3645e97645805ba77db32c433dce&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/a31cc114a89a3645e97645805ba77db32c433dce&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="824888" author="igor" created="Mon, 9 Feb 2015 16:31:42 +0000"  >&lt;p&gt;I agree that this is something we will probably want in the final product. However, it&apos;s not on the top of my TODO list just yet. I&apos;m currently running snapshot&amp;amp;replay with Parse&apos;s production workload and fixing what breaks (performance-wise and stability-wise) &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/p&gt;</comment>
                            <comment id="824722" author="dan@10gen.com" created="Mon, 9 Feb 2015 14:03:14 +0000"  >&lt;p&gt;I would strongly consider using one CF per database, this will work for parse&apos;s workload and leave some flexibility for tuning.  Also allows you to not special case the oplog.&lt;/p&gt;</comment>
                            <comment id="824616" author="igor" created="Mon, 9 Feb 2015 08:54:12 +0000"  >&lt;p&gt;It&apos;s actually one column family per mongodb instance. We could make it per database in the future.&lt;/p&gt;

&lt;p&gt;The biggest tradeoff is that dropping an index/collection might be slow. We will not be able to reclaim the space immediately, while with CF-per-collection, we would just unlink the files. We will also pay some CPU cost to prefix/unprefix all the keys, but I&apos;m not too concerned about that.&lt;/p&gt;

&lt;p&gt;We also lose some flexibility, since different column families can have different parameters, so with CF-per-collection, we can optimize different collections based on access patterns.&lt;/p&gt;

&lt;p&gt;The enabling factor for this patch was great work on KeyString. Previously we needed to have different comparators for different indexes (based on ordering parameter), which was hard to achieve with everything stored in a single column family. Now that all the keys are byte-comparable we don&apos;t have to worry about that.&lt;/p&gt;

&lt;p&gt;I am yet to optimize oplog. I might need to extract it to a separate column family, since it&apos;s a big factor in overall system performance. But this work will probably wait for MongoDB 3.2.&lt;/p&gt;</comment>
                            <comment id="824556" author="dan@10gen.com" created="Mon, 9 Feb 2015 05:16:49 +0000"  >&lt;p&gt;So it will now be one column family per database.  Are there any tradeoffs?&lt;/p&gt;

&lt;p&gt;I&apos;ll ask &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=benety.goh&quot; class=&quot;user-hover&quot; rel=&quot;benety.goh&quot;&gt;benety.goh&lt;/a&gt; to take a look at the PR&apos;s you have queued up tmrw.&lt;/p&gt;</comment>
                            <comment id="824543" author="igor" created="Mon, 9 Feb 2015 04:26:14 +0000"  >&lt;p&gt;Yes, IIRC Parse tested WiredTiger and this was one of the major issues that came up (file per collection/index). RocksDB with column families is even worse because we have multiple files per column family. There is also inherent issue with RocksDBs architecture when scaling column families (write-buffer memory management, write-ahead-log management, bigger write amplification due to potentially small Level 0 files...).&lt;/p&gt;

&lt;p&gt;Here&apos;s the patch for the change: &lt;a href=&quot;https://github.com/mongodb-partners/mongo/tree/singlecolumnfamily&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb-partners/mongo/tree/singlecolumnfamily&lt;/a&gt;. I&apos;m not sure what&apos;s the status of 3.0, but it would be great if we could also get this patch in.&lt;/p&gt;

&lt;p&gt;Instead of creating a column family for collection or an index, we assign a new prefix, which is 32-bit integer. All key-values belonging to that collection will be written as &amp;lt;prefix&amp;gt;key-value. When a collection needs to read a RecordId, it will issue request Get(&amp;lt;prefix&amp;gt;&amp;lt;record_id&amp;gt;) to get the document. Is this a bit clearer description?&lt;/p&gt;

&lt;p&gt;Here&apos;s an example of Iterator that automatically adds and removes prefixes from underlying data: &lt;a href=&quot;https://github.com/mongodb-partners/mongo/commit/5dbf6e32765cddb9173c3bd17c016680ec3fddc3#diff-a1ab21c960bbf18d99e36b078fb18718R51&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb-partners/mongo/commit/5dbf6e32765cddb9173c3bd17c016680ec3fddc3#diff-a1ab21c960bbf18d99e36b078fb18718R51&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="824537" author="dan@10gen.com" created="Mon, 9 Feb 2015 04:09:42 +0000"  >&lt;p&gt;This may be an issue for WiredTiger because all indexes and collections are stored in their own files, whereas mmap stored all collections and indexes in a single set of database files.&lt;/p&gt;

&lt;p&gt;Can you describe the proposed change a bit further?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="183226">SERVER-17223</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12451" key="com.atlassian.jira.plugin.system.customfieldtypes:multiversion">
                        <customfieldname>Backport Completed</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="14826">3.0.0-rc9</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 9 Feb 2015 04:09:42 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        8 years, 51 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_14262" key="com.atlassian.jira.plugin.system.customfieldtypes:datepicker">
                        <customfieldname>End date</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 10 Feb 2015 23:59:59 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>crystal.horn@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            8 years, 51 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>benety.goh@mongodb.com</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>igor</customfieldvalue>
            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrlcqn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hs65bj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>161051</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_14261" key="com.atlassian.jira.plugin.system.customfieldtypes:datepicker">
                        <customfieldname>Start date</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 9 Feb 2015 00:00:00 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11861" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>User Summary</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="11856"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hril7j:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>