<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:05:18 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-24076] Excess memory use on PPC64</title>
                <link>https://jira.mongodb.org/browse/SERVER-24076</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Mongod crashes while running YCSB.  Log and mongstat are attached.&lt;br/&gt;
The system keeps on consuming memory until it starts paging.&lt;br/&gt;
At the end of the log it shows:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-05-03T19:52:39.604-0400 E STORAGE  [conn35] WiredTiger (0) [1462319559:601568][62565:0x3fff98aeeb50], file:index-3-2753723832870039369.wt, WT_CURSOR.search: read checksum error for 12288B block at offset 3836055552: block header checksum of 892942393 doesn&apos;t match expected checksum of 2861721176&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-05-03T19:52:39.604-0400 E STORAGE  [conn35] WiredTiger (0) [1462319559:604949][62565:0x3fff98aeeb50], file:index-3-2753723832870039369.wt, WT_CURSOR.search: index-3-2753723832870039369.wt: encountered an illegal file format or internal value&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-05-03T19:52:39.604-0400 E STORAGE  [conn35] WiredTiger (-31804) [1462319559:604971][62565:0x3fff98aeeb50], file:index-3-2753723832870039369.wt, WT_CURSOR.search: the process must exit and restart: WT_PANIC: WiredTiger library panic&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="284926">SERVER-24076</key>
            <summary>Excess memory use on PPC64</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="michael.cahill@mongodb.com">Michael Cahill</assignee>
                                    <reporter username="lilianr@us.ibm.com">Lilian Romero</reporter>
                        <labels>
                    </labels>
                <created>Fri, 6 May 2016 03:37:56 +0000</created>
                <updated>Mon, 6 Jun 2016 21:16:30 +0000</updated>
                            <resolved>Fri, 3 Jun 2016 03:00:11 +0000</resolved>
                                    <version>3.3.5</version>
                                    <fixVersion>3.3.8</fixVersion>
                                    <component>WiredTiger</component>
                                        <votes>1</votes>
                                    <watches>14</watches>
                                                                                                                <comments>
                            <comment id="1283087" author="michael.cahill" created="Fri, 3 Jun 2016 03:00:11 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lilianr%40us.ibm.com&quot; class=&quot;user-hover&quot; rel=&quot;lilianr@us.ibm.com&quot;&gt;lilianr@us.ibm.com&lt;/a&gt;, in my testing, this change reduces the tcmalloc overhead on PPC by over 50%.  Please let me know if this does not resolve the issue for you.&lt;/p&gt;</comment>
                            <comment id="1283086" author="xgen-internal-githook" created="Fri, 3 Jun 2016 02:58:25 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;michaelcahill&apos;, u&apos;name&apos;: u&apos;Michael Cahill&apos;, u&apos;email&apos;: u&apos;michael.cahill@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-24076&quot; title=&quot;Excess memory use on PPC64&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-24076&quot;&gt;&lt;del&gt;SERVER-24076&lt;/del&gt;&lt;/a&gt; Reduce tcmalloc overhead with &amp;gt;4KB pages.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/87010b47738fe11e1395f913bfffa9e787014c96&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/87010b47738fe11e1395f913bfffa9e787014c96&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1277450" author="michael.cahill" created="Fri, 27 May 2016 12:20:14 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lilianr%40us.ibm.com&quot; class=&quot;user-hover&quot; rel=&quot;lilianr@us.ibm.com&quot;&gt;lilianr@us.ibm.com&lt;/a&gt;, I have created a patch build, the binaries are here:&lt;/p&gt;

&lt;p&gt;  &lt;a href=&quot;https://s3.amazonaws.com/mciuploads/mongodb-mongo-master/enterprise-rhel-62-64-bit/cc6ee9cf116853289ee41784220b3ce8ed14c29c/binaries/mongo-mongodb_mongo_master_enterprise_rhel_62_64_bit_cc6ee9cf116853289ee41784220b3ce8ed14c29c_16_05_26_06_54_11.tgz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://s3.amazonaws.com/mciuploads/mongodb-mongo-master/enterprise-rhel-62-64-bit/cc6ee9cf116853289ee41784220b3ce8ed14c29c/binaries/mongo-mongodb_mongo_master_enterprise_rhel_62_64_bit_cc6ee9cf116853289ee41784220b3ce8ed14c29c_16_05_26_06_54_11.tgz&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Please re-run tests with these binaries: if they avoid the excess memory use then these changes will be in a future release of MongoDB.&lt;/p&gt;</comment>
                            <comment id="1275817" author="lilianr@us.ibm.com" created="Thu, 26 May 2016 03:37:41 +0000"  >&lt;p&gt;Can you provide a fix for a non-release build? We are currently using 3.3.6.&lt;/p&gt;</comment>
                            <comment id="1270667" author="alexander.gorrod" created="Fri, 20 May 2016 21:31:01 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lilianr%40us.ibm.com&quot; class=&quot;user-hover&quot; rel=&quot;lilianr@us.ibm.com&quot;&gt;lilianr@us.ibm.com&lt;/a&gt; We will keep you updated on the status. The patch we tested will need to go through code review and testing before it makes it into a release. Let me know if the problem is blocking you from making progress in the mean time and you are interested in getting a non-release build to test with. &lt;/p&gt;</comment>
                            <comment id="1270214" author="lilianr@us.ibm.com" created="Fri, 20 May 2016 16:15:35 +0000"  >&lt;p&gt;The error that I was able to reproduce was the memory issue. I have not been able to re-create the data corruption again. Let me know in which build the memory issue is resolved. &lt;br/&gt;
For now, lets defer the data corruption issue until we are able to reproduce it again. If reproducible then I will leave the system as is so we can debug it. &lt;/p&gt;</comment>
                            <comment id="1269570" author="alexander.gorrod" created="Thu, 19 May 2016 23:28:33 +0000"  >&lt;blockquote&gt;&lt;p&gt;At the beginning of the workload phase the diagnostic data shows that the allocated memory grows larger than the configured cache size.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;We have figured out that this problem is specific to PPC, and is due to some changes we have made to optimize tcmalloc in the development version of MongoDB, that we have not yet fully tested.&lt;/p&gt;

&lt;p&gt;The technical problem is that we divide memory allocations into two different categories: allocations less than 16k and allocations greater than 16k. Allocations greater than 16k are done using a different allocation scheme. On PPC platforms, the minimum allocation size for allocations in the second allocation scheme is 64k to match the OS virtual memory page size. During the workload phase WiredTiger does a lot of allocations in the 28k-&amp;gt;32k range, which will result an overhead of 50% in the system allocator.&lt;/p&gt;

&lt;p&gt;We have tested a change to our allocator that reduces the overhead for larger allocations on PPC. The following graph demonstrates the behavior before (first half) and after (second half) our change. You can see that &quot;tcmalloc allocated minus wt cache&quot; remains stable in the second run, wheras it jumps up during the workload phase of the first run.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/122546/Screen%20Shot%202016-05-19%20at%206.54.57%20pm.png&quot; width=&quot;600&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="1269562" author="alexander.gorrod" created="Thu, 19 May 2016 23:11:24 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lilianr%40us.ibm.com&quot; class=&quot;user-hover&quot; rel=&quot;lilianr@us.ibm.com&quot;&gt;lilianr@us.ibm.com&lt;/a&gt; Sorry for the radio silence on this one - we have been digging deeper. We have identified three different interesting characteristics from this ticket so far:&lt;/p&gt;

&lt;p&gt;1) At the beginning of the workload phase the diagnostic data shows that the allocated memory grows larger than the configured cache size.&lt;br/&gt;
2) We see that there is memory overhead being held in the memory allocator.&lt;br/&gt;
3) The WT_PANIC you initially reported. We have not been able to make progress on that one, since it does not appear to be reproducible.&lt;/p&gt;

&lt;p&gt;We understand the first two problems, and I&apos;ll address them in separate comments below. For the third problem - do you have any ideas how we can proceed? I&apos;m not sure how to make progress.&lt;/p&gt;</comment>
                            <comment id="1269548" author="alexander.gorrod" created="Thu, 19 May 2016 22:56:55 +0000"  >&lt;p&gt;tcmalloc 64k page allocation overhead&lt;/p&gt;</comment>
                            <comment id="1263968" author="alexander.gorrod" created="Fri, 13 May 2016 22:47:54 +0000"  >&lt;p&gt;Thanks for attaching the additional data &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lilianr%40us.ibm.com&quot; class=&quot;user-hover&quot; rel=&quot;lilianr@us.ibm.com&quot;&gt;lilianr@us.ibm.com&lt;/a&gt;. I&apos;ve taken a quick look and can see some differences - I&apos;ll dig deeper into analysing the data over the next few days and update the ticket when I have useful information.&lt;/p&gt;

&lt;p&gt;I have attempted to reproduce the behavior you reported on a non-PPC machine, and have not had any luck. Could you share some more information about the YCSB setup you are using? Especially:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;Which version of the YCSB benchmark you are running&lt;/li&gt;
	&lt;li&gt;How many threads you are configuring for the load and run&lt;/li&gt;
	&lt;li&gt;Whether you are using an explicit &lt;tt&gt;requestdistribution&lt;/tt&gt; setting for YCSB, or the default uniform&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Thanks for your help in tracking this problem down.&lt;/p&gt;

&lt;p&gt;Alex&lt;/p&gt;</comment>
                            <comment id="1263892" author="lilianr@us.ibm.com" created="Fri, 13 May 2016 20:52:11 +0000"  >&lt;p&gt;The attachment contains the diagnostic data for 3.2.6&lt;/p&gt;</comment>
                            <comment id="1262445" author="lilianr@us.ibm.com" created="Wed, 11 May 2016 23:55:39 +0000"  >&lt;p&gt;To clarify, the problem that is easy to reproduce is the excess use of memory compared to 3.2.6.  Have not seen the checksum mismatch in 3.2.6. I will post diagnostics data for 3.2.6 by tomorrow. Will  re-run with 3.3.5 to find out if the checksum mismatch can be reproduced.&lt;/p&gt;</comment>
                            <comment id="1261842" author="michael.cahill" created="Wed, 11 May 2016 12:32:33 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lilianr%40us.ibm.com&quot; class=&quot;user-hover&quot; rel=&quot;lilianr@us.ibm.com&quot;&gt;lilianr@us.ibm.com&lt;/a&gt;, I will try to reproduce on a test system today and let you know what I find.  Please note that I am out of the office at meetings this week so I may not be able to respond immediately to updates.&lt;/p&gt;

&lt;p&gt;I can see from the data you have uploaded that the excess memory use is a combination of memory allocator fragmentation (44GB) and memory allocated outside the WiredTiger cache (39GB).  The former is unfortunate but we understand it and have some workarounds, the latter is unexpected and I will try to understand what is causing it.&lt;/p&gt;

&lt;p&gt;If you can post diagnostic data from a 3.2.6 run, that would be very helpful.  Can you also clarify whether the checksum mismatch is occurring with 3.2.6?&lt;/p&gt;</comment>
                            <comment id="1257699" author="lilianr@us.ibm.com" created="Fri, 6 May 2016 15:16:59 +0000"  >&lt;p&gt;attaching diagnostic data. &lt;br/&gt;
I tried the same test using mongodb3.2.6 and don&apos;t run into the problem where it keeps on consuming memory. So there may be 2 issues: 1) memory leak and 2) data corruption. Let me know if you need the mongostat from3.2.6 to compare.&lt;br/&gt;
The disk subsystem is not reporting any errors.&lt;/p&gt;</comment>
                            <comment id="1257328" author="alexander.gorrod" created="Fri, 6 May 2016 05:52:41 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lilianr%40us.ibm.com&quot; class=&quot;user-hover&quot; rel=&quot;lilianr@us.ibm.com&quot;&gt;lilianr@us.ibm.com&lt;/a&gt; I&apos;m sorry to hear that you have encountered a problem using MongoDB.&lt;/p&gt;

&lt;p&gt;The most common cause for a checksum error is a disk corruption; are you running this test using a reliable disk subsystem? Does the failure reproduce reliably?&lt;/p&gt;

&lt;p&gt;You mention that &lt;tt&gt;The system keeps on consuming memory until it starts paging&lt;/tt&gt;. That is unfortunate - I see that mongod is using a WiredTiger cache size of 60GB. Do you have either multiple mongod instances running on the same machine, or other processes running on the same machine that are consuming a lot of memory as well?&lt;/p&gt;

&lt;p&gt;It would be very useful if you can also upload the content of the diagnostic.data directory that should be being created as a subdirectory of the MongoDB database directory.&lt;/p&gt;

&lt;p&gt;One more request: Could you try running the &lt;a href=&quot;https://docs.mongodb.com/manual/reference/command/validate/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;validate &lt;/a&gt; command on the collections in the database, and report back the status of running that command?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="287805">SERVER-24207</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="122546" name="Screen Shot 2016-05-19 at 6.54.57 pm.png" size="60238" author="alexander.gorrod@mongodb.com" created="Thu, 19 May 2016 22:56:55 +0000"/>
                            <attachment id="120790" name="diagnostic.data.tar.gz" size="955279" author="lilianr@us.ibm.com" created="Fri, 6 May 2016 15:16:59 +0000"/>
                            <attachment id="121795" name="diagnostic.data_3.2.6.tar.gz" size="3940021" author="lilianr@us.ibm.com" created="Fri, 13 May 2016 20:52:11 +0000"/>
                            <attachment id="120738" name="mongod.out.gz" size="6395931" author="lilianr@us.ibm.com" created="Fri, 6 May 2016 03:37:56 +0000"/>
                            <attachment id="120739" name="mongostat.out" size="13106" author="lilianr@us.ibm.com" created="Fri, 6 May 2016 03:37:56 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>15.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 6 May 2016 05:52:41 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 36 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 36 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>lilianr@us.ibm.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrk8fr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hsl5xj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;version 3.3.5 for ppc64le downloaded from the website&lt;br/&gt;
OS: RHEL7.2 , Memory: 128GB&lt;br/&gt;
Workload: YCSB&lt;br/&gt;
Used the following:&lt;br/&gt;
fieldcount=10&lt;br/&gt;
fieldlength=10&lt;br/&gt;
recordcount=120000000&lt;br/&gt;
operationcount=100000000&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hriyyf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>