<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:02:18 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-3177] Massive Performance hit on index creation when doing a db.copyDatabase or a mongorestore on a server with a replicated set configuration</title>
                <link>https://jira.mongodb.org/browse/SERVER-3177</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;When restoring a large collection on a mongodb server with a replicated set configuration, index creation becomes incredibly slow.  In our case, the collection index creation could take anywhere from between 20 minutes to over half an hour&lt;/p&gt;

&lt;p&gt;The exact same collection restored to a test mongodb server running inside a virtual machine with much smaller specs, but with no replication sets, the same indexes are created in 5 to 6 seconds.&lt;/p&gt;

&lt;p&gt;Examing the log files, we can see the following:&lt;/p&gt;
&lt;blockquote&gt;
&lt;p&gt;// log file from big server, replication sets, secondary server on&lt;br/&gt;
Wed Jun  1 03:37:31 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; building new index on &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { apps.aid}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt; for cassia.snapshots_2011053100_01&lt;br/&gt;
                239000/326021   73%&lt;br/&gt;
                259900/326021   79%&lt;br/&gt;
                268900/326021   82%&lt;br/&gt;
                277600/326021   85%&lt;br/&gt;
                287400/326021   88%&lt;br/&gt;
                289600/326021   88%&lt;br/&gt;
                300000/326021   92%&lt;br/&gt;
                310000/326021   95%&lt;br/&gt;
                311900/326021   95%&lt;br/&gt;
                314200/326021   96%&lt;br/&gt;
                316500/326021   97%&lt;br/&gt;
                318400/326021   97%&lt;br/&gt;
                320500/326021   98%&lt;br/&gt;
                322600/326021   98%&lt;br/&gt;
                324900/326021   99%&lt;br/&gt;
Wed Jun  1 03:40:28 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt;     done building bottom layer, going to commit&lt;br/&gt;
Wed Jun  1 03:40:37 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; done for 1824 records 185.837secs&lt;br/&gt;
Wed Jun  1 03:40:37 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; insert cassia.system.indexes 185837ms&lt;br/&gt;
Wed Jun  1 03:40:37 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 79905899&lt;br/&gt;
Wed Jun  1 03:40:37 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; building new index on &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { cnt}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt; for cassia.snapshots_2011053100_02&lt;br/&gt;
Wed Jun  1 03:40:39 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; done for 1823 records 2.376secs&lt;br/&gt;
Wed Jun  1 03:40:39 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; insert cassia.system.indexes 2376ms&lt;br/&gt;
Wed Jun  1 03:40:39 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; building new index on &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { apps.aid}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt; for cassia.snapshots_2011053100_02&lt;br/&gt;
                100/1823        5%&lt;br/&gt;
                200/1823        10%&lt;br/&gt;
                300/1823        16%&lt;br/&gt;
                1000/1823       54%&lt;br/&gt;
                1100/1823       60%&lt;br/&gt;
                1200/1823       65%&lt;br/&gt;
                1300/1823       71%&lt;br/&gt;
                1800/1823       98%&lt;br/&gt;
Wed Jun  1 03:54:09 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt;     external sort used : 170 files  in 809 secs&lt;br/&gt;
                100/325822      0%&lt;br/&gt;
                126600/325822   38%&lt;br/&gt;
                128200/325822   39%&lt;br/&gt;
                129500/325822   39%&lt;br/&gt;
                130200/325822   39%&lt;br/&gt;
                131300/325822   40%&lt;br/&gt;
                131900/325822   40%&lt;br/&gt;
                132900/325822   40%&lt;br/&gt;
                133600/325822   41%&lt;br/&gt;
                134100/325822   41%&lt;br/&gt;
                134900/325822   41%&lt;br/&gt;
                135900/325822   41%&lt;br/&gt;
                136800/325822   41%&lt;br/&gt;
                137900/325822   42%&lt;br/&gt;
                138800/325822   42%&lt;br/&gt;
                139900/325822   42%&lt;br/&gt;
                140700/325822   43%&lt;br/&gt;
                141400/325822   43%&lt;br/&gt;
                141900/325822   43%&lt;br/&gt;
                142900/325822   43%&lt;br/&gt;
                143800/325822   44%&lt;br/&gt;
                144900/325822   44%&lt;br/&gt;
                145800/325822   44%&lt;br/&gt;
                146700/325822   45%&lt;br/&gt;
                148300/325822   45%&lt;br/&gt;
                149800/325822   45%&lt;br/&gt;
                150200/325822   46%&lt;br/&gt;
                305200/325822   93%&lt;br/&gt;
Wed Jun  1 03:59:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt;     done building bottom layer, going to commit&lt;br/&gt;
Wed Jun  1 03:59:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; done for 1823 records 1140.97secs&lt;br/&gt;
Wed Jun  1 03:59:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn16&amp;#93;&lt;/span&gt; insert cassia.system.indexes 1140969ms&lt;br/&gt;
Wed Jun  1 03:59:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn113&amp;#93;&lt;/span&gt; getmore cassia.system.namespaces cid:6678415019539336828 getMore: {}  bytes:77862 nreturned:1396 1355345ms&lt;br/&gt;
Wed Jun  1 03:59:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn99&amp;#93;&lt;/span&gt; query cassia.snapshots_2011060101_02 ntoreturn:1 reslen:44018 nscanned:1 &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { cnt}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;  nreturned:1 1143466ms&lt;br/&gt;
Wed Jun  1 03:59:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 81033909&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;The log files from the small server, single mongo, no replicated sets shows the following:&lt;/p&gt;
&lt;blockquote&gt;
&lt;p&gt;Wed Jun  1 03:53:35 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; building new index on &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { apps.aid}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt; for cassia.snapshots_2011053100_02&lt;br/&gt;
Wed Jun  1 03:53:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; done for 1823 records 5.612secs&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;As a test, I shutdown the secondary server while the restore was happening and the index creation time on the primary server when down to a much more acceptable 5 to 6 seconds.&lt;/p&gt;


&lt;p&gt;More datapoints:&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;using the db.currentOp() command shows the index creation as &quot;external sort used&quot; on the primary server with the secondary server active&lt;/li&gt;
	&lt;li&gt;that also appears as &quot;external sort used : 170 files  in 809 secs&quot; in the log file&lt;/li&gt;
	&lt;li&gt;after shutting down the secondary server, the &quot;external sort used&quot; goes away&lt;/li&gt;
	&lt;li&gt;&quot;external sorts&quot; never appear in the log files on the same data set in the test database.&lt;/li&gt;
	&lt;li&gt;the slow performance occurs whether I use the mongorestore command or the db.copyDatabase(&quot;old collection&quot;, &quot;new collection&quot;, &quot;old server&quot;) command&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment>Ubuntu server 10.04.2 LTS&lt;br/&gt;
12 GB ram&lt;br/&gt;
2 x raid 1 volumes, data on one volume, journal on another.&lt;br/&gt;
2 x 6 core 2.9 GHz cpu with hyperthreading (24 logical cpus)</environment>
        <key id="17737">SERVER-3177</key>
            <summary>Massive Performance hit on index creation when doing a db.copyDatabase or a mongorestore on a server with a replicated set configuration</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="5">Cannot Reproduce</resolution>
                                        <assignee username="aaron">Aaron Staple</assignee>
                                    <reporter username="pdwalker">Paul D. Walker</reporter>
                        <labels>
                            <label>indexing</label>
                            <label>performance</label>
                            <label>replication</label>
                    </labels>
                <created>Wed, 1 Jun 2011 07:33:10 +0000</created>
                <updated>Fri, 30 Mar 2012 14:37:00 +0000</updated>
                            <resolved>Wed, 8 Jun 2011 14:28:46 +0000</resolved>
                                    <version>1.8.1</version>
                                                    <component>Index Maintenance</component>
                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>2</watches>
                                                                                                                <comments>
                            <comment id="36670" author="pdwalker" created="Wed, 8 Jun 2011 11:40:58 +0000"  >&lt;p&gt;Hi Aaron, &lt;/p&gt;

&lt;p&gt;I think I&apos;ve found the problem.&lt;/p&gt;

&lt;p&gt;I was restoring about 600 collections, each collection of generally about 70 MB in size.  The total size of the collections &amp;gt;&amp;gt; system memory. When restoring the the database with all those collections, the initial collections all fit in ram, but later on performance started to suffer.  I was expecting mongo to free up the memory the least used collections and keep enough to continue fast indexing&lt;/p&gt;

&lt;p&gt;I did see a performance increase after shutting down the secondary, but that may have been just coincidence. I would need to test it more fully to be 100% sure that this was really the case, or whether I happened to be indexing one of the smaller collections at that moment.&lt;/p&gt;

&lt;p&gt;I&apos;ve reset up my replicated set configuration and loaded smaller subsets of the collections ( size &amp;lt; system ram ) and there have been no problems with performance. &lt;/p&gt;

&lt;p&gt;I&apos;m also going to run a test by loading the collections individually until the db collections size is &amp;gt; system.ram and see if my problems return, or if mongodb &quot;handles&quot; it a little better.&lt;/p&gt;

&lt;p&gt;Summary: I think it turns out to be my fault.&lt;/p&gt;</comment>
                            <comment id="36609" author="aaron" created="Wed, 8 Jun 2011 03:10:35 +0000"  >&lt;p&gt;Hi Paul,&lt;/p&gt;

&lt;p&gt;We haven&apos;t been able to reproduce this issue. Can you supply your mongod logs for the primary and secondary?&lt;/p&gt;</comment>
                            <comment id="36599" author="pdwalker" created="Wed, 8 Jun 2011 02:09:17 +0000"  >&lt;p&gt;any other questions or information I can provide?&lt;/p&gt;</comment>
                            <comment id="35704" author="pdwalker" created="Thu, 2 Jun 2011 11:59:25 +0000"  >&lt;p&gt;The secondary has an identical hardware configuration to the primary.&lt;/p&gt;</comment>
                            <comment id="35703" author="pdwalker" created="Thu, 2 Jun 2011 11:58:39 +0000"  >&lt;p&gt;additional datapoint&lt;/p&gt;

&lt;p&gt;The servers are now running and new collections added to the primary are being indexed in memory and quickly:&lt;/p&gt;

&lt;p&gt;Thu Jun  2 11:47:50 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn118&amp;#93;&lt;/span&gt; building new index on &lt;/p&gt;
{ _id: 1 }
&lt;p&gt; for cassia.snapshots_2011060211_03&lt;br/&gt;
Thu Jun  2 11:47:50 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn118&amp;#93;&lt;/span&gt; done for 0 records 0secs&lt;br/&gt;
Thu Jun  2 11:47:50 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn118&amp;#93;&lt;/span&gt; building new index on &lt;/p&gt;
{ cnt: 1, ty: 1, cat: 1 }
&lt;p&gt; for cassia.snapshots_2011060211_03&lt;br/&gt;
Thu Jun  2 11:47:50 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn118&amp;#93;&lt;/span&gt; done for 1 records 0secs&lt;br/&gt;
Thu Jun  2 11:47:50 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn118&amp;#93;&lt;/span&gt; building new index on &lt;/p&gt;
{ apps.aid: 1 }
&lt;p&gt; for cassia.snapshots_2011060211_03&lt;br/&gt;
Thu Jun  2 11:47:50 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn118&amp;#93;&lt;/span&gt; done for 1 records 0.001secs&lt;/p&gt;

&lt;p&gt;However the secondary server is still catching up to the primary.  It&apos;s taking between 5 to 15 minutes for these indexes to be created&lt;/p&gt;
</comment>
                            <comment id="35702" author="pdwalker" created="Thu, 2 Jun 2011 11:54:32 +0000"  >&lt;p&gt;No, the disks are almost completely idle when this is happening.&lt;/p&gt;

&lt;p&gt;(sda contains the mongo journals, sdb contains the data dir)&lt;/p&gt;

&lt;p&gt;mdrop@obdb1:~$ iostat -xm 2 &lt;br/&gt;
Linux 2.6.32-32-generic (obdb2.so1)     06/02/2011      &lt;em&gt;x86_64&lt;/em&gt;        (24 CPU)&lt;/p&gt;

&lt;p&gt;avg-cpu:  %user   %nice %system %iowait  %steal   %idle&lt;br/&gt;
           0.08    0.00    1.86    0.72    0.00   97.33&lt;/p&gt;

&lt;p&gt;Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await  svctm  %util&lt;br/&gt;
sda               0.08     0.90    0.22    1.76     0.01     0.31   332.08     0.01    5.93   0.89   0.18&lt;br/&gt;
sdb               0.00    81.03    2.04    3.15     0.11     0.33   171.25     0.28   54.50   0.61   0.32&lt;/p&gt;

&lt;p&gt;avg-cpu:  %user   %nice %system %iowait  %steal   %idle&lt;br/&gt;
           0.10    0.00    5.04    4.86    0.00   90.00&lt;/p&gt;

&lt;p&gt;Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await  svctm  %util&lt;br/&gt;
sda               8.00     0.00    3.00    0.00     0.04     0.00    30.67     0.01    5.00   5.00   1.50&lt;br/&gt;
sdb               0.00     2.50    2.00    1.00     0.06     0.01    48.00     0.00    0.00   0.00   0.00&lt;/p&gt;

&lt;p&gt;avg-cpu:  %user   %nice %system %iowait  %steal   %idle&lt;br/&gt;
           0.06    0.00    3.30    3.01    0.00   93.64&lt;/p&gt;

&lt;p&gt;Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await  svctm  %util&lt;br/&gt;
sda               5.00     1.50    1.00    1.00     0.02     0.01    34.00     0.01    5.00   5.00   1.00&lt;br/&gt;
sdb               0.00     8.50    0.00    0.50     0.00     0.04   144.00     0.01   10.00  10.00   0.50&lt;/p&gt;

&lt;p&gt;avg-cpu:  %user   %nice %system %iowait  %steal   %idle&lt;br/&gt;
           0.09    0.00    5.79    1.49    0.00   92.63&lt;/p&gt;

&lt;p&gt;Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await  svctm  %util&lt;br/&gt;
sda               0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00   0.00   0.00&lt;br/&gt;
sdb               0.00     0.00    2.00    0.00     0.06     0.00    58.00     0.00    0.00   0.00   0.00&lt;/p&gt;

&lt;p&gt;avg-cpu:  %user   %nice %system %iowait  %steal   %idle&lt;br/&gt;
           0.02    0.00    4.61    0.00    0.00   95.36&lt;/p&gt;

&lt;p&gt;Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await  svctm  %util&lt;br/&gt;
sda               0.00     1.00    0.00    5.00     0.00     0.02     9.60     0.00    0.00   0.00   0.00&lt;br/&gt;
sdb               0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00   0.00   0.00&lt;/p&gt;

&lt;p&gt;avg-cpu:  %user   %nice %system %iowait  %steal   %idle&lt;br/&gt;
           0.02    0.00    4.77    0.00    0.00   95.21&lt;/p&gt;

&lt;p&gt;Device:         rrqm/s   wrqm/s     r/s     w/s    rMB/s    wMB/s avgrq-sz avgqu-sz   await  svctm  %util&lt;br/&gt;
sda               0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00   0.00   0.00&lt;br/&gt;
sdb               0.00     1.50    2.00    1.00     0.06     0.01    45.33     0.00    0.00   0.00   0.00&lt;/p&gt;</comment>
                            <comment id="35571" author="eliot" created="Wed, 1 Jun 2011 15:37:23 +0000"  >&lt;p&gt;Is the disk saturated?&lt;br/&gt;
Can you run iostat -x 2 while this is happening?&lt;/p&gt;</comment>
                            <comment id="35504" author="pdwalker" created="Wed, 1 Jun 2011 10:44:15 +0000"  >&lt;p&gt;additional datapoints&lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;after the database is loaded, new collections of a similar size are indexed quickly in the primary&lt;/li&gt;
&lt;/ul&gt;


&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;the secondary continues to use external sorts, so is slow catching up&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="35470" author="pdwalker" created="Wed, 1 Jun 2011 07:34:11 +0000"  >&lt;p&gt;If you need any more information for debugging purposes, I&apos;ll be happy to provide it.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 1 Jun 2011 15:37:23 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        12 years, 37 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ian@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            12 years, 37 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>aaron</customfieldvalue>
            <customfieldvalue>eliot</customfieldvalue>
            <customfieldvalue>pdwalker</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hroyv3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrifpb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>21196</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht0c1j:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>