<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:05:51 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-24266] Large discrepancy in SizeStorer after unclean shutdown</title>
                <link>https://jira.mongodb.org/browse/SERVER-24266</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The &lt;tt&gt;mongod&lt;/tt&gt; instance (with WiredTiger storage engine) is down when there is insufficient disk space for the data volume. After adding more space to the dbpath and restart the &lt;tt&gt;mongod&lt;/tt&gt; instance, &lt;tt&gt;count()&lt;/tt&gt; returns incorrect value as below:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&amp;gt; db.users.count()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;1527296&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&amp;gt; db.users.find().itcount()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;1605004&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-19472&quot; title=&quot;count() incorrect after recovery with WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-19472&quot;&gt;&lt;del&gt;SERVER-19472&lt;/del&gt;&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/browse/DOCS-5682&quot; title=&quot;Document that the size and count in collection stats is not guaranteed to be 100% accurate&quot; class=&quot;issue-link&quot; data-issue-key=&quot;DOCS-5682&quot;&gt;&lt;del&gt;DOCS-5682&lt;/del&gt;&lt;/a&gt; mentioned that count may off by up to 1000 documents after an unclean shutdown. However, count off to 70k+ documents in my test.&lt;/p&gt;

&lt;p&gt;Have tested 3.0.12, 3.2.6 and 3.3.6, all have the same issue.&lt;/p&gt;

&lt;p&gt;&lt;tt&gt;db.collection.validate(&lt;b&gt;true&lt;/b&gt;)&lt;/tt&gt; will reset the value for &lt;tt&gt;count()&lt;/tt&gt; to the correct number.&lt;/p&gt;</description>
                <environment></environment>
        <key id="288868">SERVER-24266</key>
            <summary>Large discrepancy in SizeStorer after unclean shutdown</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="10038" iconUrl="https://jira.mongodb.org/images/icons/subtask.gif" description="">Backlog</status>
                    <statusCategory id="2" key="new" colorName="default"/>
                                    <resolution id="-1">Unresolved</resolution>
                                        <assignee username="backlog-server-execution">Backlog - Storage Execution Team</assignee>
                                    <reporter username="linda.qin@mongodb.com">Linda Qin</reporter>
                        <labels>
                            <label>pm-1820</label>
                    </labels>
                <created>Tue, 24 May 2016 02:17:29 +0000</created>
                <updated>Sat, 10 Jun 2023 04:16:28 +0000</updated>
                                            <version>3.0.12</version>
                    <version>3.2.6</version>
                    <version>3.3.6</version>
                                                    <component>WiredTiger</component>
                                        <votes>1</votes>
                                    <watches>25</watches>
                                                                                                                <comments>
                            <comment id="1280404" author="michael.cahill" created="Wed, 1 Jun 2016 00:34:36 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=ravind.kumar&quot; class=&quot;user-hover&quot; rel=&quot;ravind.kumar&quot;&gt;ravind.kumar&lt;/a&gt;, yes, that is a more accurate statement.&lt;/p&gt;

&lt;p&gt;The issue is the updates performed during storage engine recovery (or with &lt;tt&gt;nojournal&lt;/tt&gt;, the updates in between the last sizeStorer sync and the final checkpoint).  Those updates are not reflected in the &quot;size storer&quot; table (because the storage engine doesn&apos;t know that size storer is special).  So the maximum drift is determined by the time since the last flush of the size storer (every 60 seconds assuming there is any storage engine activity at all) combined with the the time since the last checkpoint completed (and checkpoints are triggered every 60 seconds by default).&lt;/p&gt;

&lt;p&gt;The two conditions make the 60 second upper bound somewhat fuzzy, but it should hold in all but the most extreme cases.&lt;/p&gt;</comment>
                            <comment id="1279596" author="ravind.kumar" created="Tue, 31 May 2016 14:36:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt;, would it be accurate to say &quot;size and count may drift by the number of updates performed within up to 60 seconds of the unclean shutdown.&quot; ? I&apos;m assuming that, if there was an unclean shutdown at T = 61, updates at T = 1 would have been fine, and if no writes occurred from T = 1 to T = 60, then there generally should not be any drift. Assuming the flush period synced exactly with the unclean shutdown.&lt;/p&gt;</comment>
                            <comment id="1276450" author="dan@10gen.com" created="Thu, 26 May 2016 16:30:20 +0000"  >&lt;p&gt;Yes.  That restores the correct size and count information.  There is a slight change in that starting in 3.2, it is no longer required to run a &quot;full&quot; validate to update the size/count info.  See: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-20286&quot; title=&quot;should update WT size storer for non-full validate&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-20286&quot;&gt;&lt;del&gt;SERVER-20286&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1276439" author="ravind.kumar" created="Thu, 26 May 2016 16:22:54 +0000"  >&lt;p&gt;The documentation currently suggests using &lt;a href=&quot;https://docs.mongodb.com/manual/reference/command/validate/#dbcmd.validate&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;tt&gt;validate&lt;/tt&gt;&lt;/a&gt;. Is this still a valid solution?&lt;/p&gt;</comment>
                            <comment id="1276418" author="dan@10gen.com" created="Thu, 26 May 2016 16:06:53 +0000"  >&lt;p&gt;I have been walking around with the idea that the sizeStorer was being flushed every 1000 docs.  Thanks for the clarification.&lt;br/&gt;
We should immediately fix the documentation (&lt;a href=&quot;https://jira.mongodb.org/browse/DOCS-7979&quot; title=&quot;Size and count in collection stats are persisted during checkpoints in WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;DOCS-7979&quot;&gt;&lt;del&gt;DOCS-7979&lt;/del&gt;&lt;/a&gt;).  We should leave this open as a placeholder to decide whether or not we want to invest effort in tightening the time between persisting to disk.&lt;/p&gt;</comment>
                            <comment id="1275857" author="michael.cahill" created="Thu, 26 May 2016 05:06:21 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=pasette&quot; class=&quot;user-hover&quot; rel=&quot;pasette&quot;&gt;pasette&lt;/a&gt;, unless I&apos;m misunderstanding the code, we update the in-memory cache every 1000 document writes, but the updates only become persistent when &lt;tt&gt;WiredTigerKVEngine::syncSizeInfo&lt;/tt&gt; is called, most commonly every 60 seconds as part of &lt;tt&gt;WiredTigerKVEngine::haveDropsQueued&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;We could get more aggressive about flushing the cache and put a time bound on this instead (e.g. every second we scan all the collections and flush any changes).  I suspect there may be issues writing to size storer as part of the unit of work that triggered the size update (since that update could abort or just stall, blocking further updates to the size storer table).&lt;/p&gt;</comment>
                            <comment id="1274822" author="dan@10gen.com" created="Wed, 25 May 2016 14:38:37 +0000"  >&lt;p&gt;The sizeStorer is meant to persist the document count and size &lt;a href=&quot;https://github.com/mongodb/mongo/blob/master/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp#L1790&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;every 1000 document writes&lt;/a&gt;.  If the journal is enabled, this should result in up to ~1000 documents drift (this is not totally exact because of the window in which the journal commits is not synchronized to the 1000 document count).  How could this condition occur where documents are written to the data store but not the sizeStorer? &lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="226468">SERVER-20063</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="289627">DOCS-7979</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1110707">SERVER-45718</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25136"><![CDATA[Storage Execution]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000XO4IhIAL, 500A000000XN5I2IAL, 500A000000Z38IkIAJ]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 24 May 2016 11:23:03 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 37 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-2944</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>dmitry.ryabtsev@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 37 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>backlog-server-execution</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>linda.qin@mongodb.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
            <customfieldvalue>ravind.kumar</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrk7cf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr24k7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;ol&gt;
	&lt;li&gt;Start a standalone &lt;tt&gt;mongod&lt;/tt&gt; instance with wiredTiger storage engine.&lt;/li&gt;
	&lt;li&gt;Insert documents to a collection until the data volume runs out of disk space.&lt;/li&gt;
	&lt;li&gt;Allocate/free more disk space for the data volume.&lt;/li&gt;
	&lt;li&gt;Start the &lt;tt&gt;mongod&lt;/tt&gt; instance again.&lt;/li&gt;
	&lt;li&gt;Check the results of &lt;tt&gt;count()&lt;/tt&gt; and &lt;tt&gt;itcount()&lt;/tt&gt; for the collection. The numbers are different.&lt;/li&gt;
&lt;/ol&gt;
</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hsexbr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>