<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:37:50 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-34783] powercycle should ensure it has the complete output from mongod before crashing the machine</title>
                <link>https://jira.mongodb.org/browse/SERVER-34783</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;While investigating &lt;a href=&quot;https://evergreen.mongodb.com/task/mongodb_mongo_v3.6_ubuntu1404_powercycle_fcv3.4_WT_4c5a0172a39f8d2dcfddee96d75ea422e63a86ea_18_04_06_17_50_22/0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this powercycle failure&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=maria.vankeulen&quot; class=&quot;user-hover&quot; rel=&quot;maria.vankeulen&quot;&gt;maria.vankeulen&lt;/a&gt; and I noticed a 2-minute gap between when the machine was crashed and when the server was restarted.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-04-07T02:36:09.802+0000 I NETWORK  [conn2] received client metadata from 127.0.0.1:49695 conn2: { driver: { name: &quot;PyMongo&quot;, version: &quot;3.5.1&quot; }, os: { type: &quot;Linux&quot;, name: &quot;debian jessie/sid&quot;, architecture: &quot;x86_64&quot;, version: &quot;3.13.0-24-generic&quot; }, platform: &quot;CPython 2.7.13.final.0&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-04-07T02:36:09.805+0000 I NETWORK  [conn2] end connection 127.0.0.1:49695 (1 connection now open)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-04-07T02:36:09.805+0000 I NETWORK  [conn1] end connection 127.0.0.1:49694 (0 connections now open)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-04-07T02:38:26.366+0000 I CONTROL  [main] ***** SERVER RESTARTED *****&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-04-07T02:38:26.452+0000 I CONTROL  [initandlisten] MongoDB starting : pid=1639 port=20001 dbpath=/data/db 64-bit host=ip-10-122-6-192&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-04-07T02:38:26.452+0000 I CONTROL  [initandlisten] db version v3.6.4-rc0-1-g4c5a017&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2018/04/07 02:38:00.632] Server crashing now&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;I believe this is behavior is &quot;as designed&quot; with how we only call &lt;tt&gt;flush()&lt;/tt&gt; on the log file and not &lt;tt&gt;fsync()&lt;/tt&gt;. This is undesirable because the mongod logs are a record of operations that happened against the database which is useful to consult when investigating a data inconsistency failure. After discussing with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt; via Slack, I think there a few ways to address this issue:&lt;/p&gt;

&lt;p&gt;&lt;del&gt;1. Change &lt;tt&gt;powertest.py&lt;/tt&gt; to send the mongod process a &lt;tt&gt;SIGUSR1&lt;/tt&gt; signal to cause it to reopen its log file (the mongod process is started with &lt;tt&gt;&amp;#45;&amp;#45;logappend&lt;/tt&gt;).&lt;/del&gt; I don&apos;t think this has the intended effect because &lt;tt&gt;close()&lt;/tt&gt; doesn&apos;t implicitly call &lt;tt&gt;fsync()&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;2. Change &lt;tt&gt;powertest.py&lt;/tt&gt; to &lt;tt&gt;rsync&lt;/tt&gt; the mongod.log file back to the controller before crashing the machine. This seems potentially tricky with how we&apos;d like for the machine to be crashed immediately after the journaled canary insert finishes.&lt;/p&gt;

&lt;p&gt;3. Add a testing-only mode where mongod opens the log file with &lt;tt&gt;O_DSYNC&lt;/tt&gt;. (&lt;a href=&quot;http://man7.org/linux/man-pages/man2/open.2.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://man7.org/linux/man-pages/man2/open.2.html&lt;/a&gt; is a handy reference.)&lt;/p&gt;

&lt;p&gt;4. Add a testing-only command that causes mongod to &lt;tt&gt;fsync()&lt;/tt&gt; its log file.&lt;/p&gt;</description>
                <environment></environment>
        <key id="537740">SERVER-34783</key>
            <summary>powercycle should ensure it has the complete output from mongod before crashing the machine</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="2">Won&apos;t Fix</resolution>
                                        <assignee username="backlog-server-stm">Backlog - Server Tooling and Methods (STM)</assignee>
                                    <reporter username="max.hirschhorn@mongodb.com">Max Hirschhorn</reporter>
                        <labels>
                            <label>tig-powercycle</label>
                    </labels>
                <created>Wed, 2 May 2018 02:53:14 +0000</created>
                <updated>Tue, 6 Dec 2022 03:30:48 +0000</updated>
                            <resolved>Fri, 5 Nov 2021 17:41:37 +0000</resolved>
                                                                    <component>Logging</component>
                    <component>Testing Infrastructure</component>
                                        <votes>0</votes>
                                    <watches>7</watches>
                                                                                                                <comments>
                            <comment id="4172208" author="brooke.miller" created="Fri, 5 Nov 2021 17:41:37 +0000"  >&lt;p&gt;We&apos;re not planning on enhancing powercycle for this. Please consider reaching out to &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=robert.guo&quot; class=&quot;user-hover&quot; rel=&quot;robert.guo&quot;&gt;robert.guo&lt;/a&gt; about using Antithesis for additional debugging information.&lt;/p&gt;</comment>
                            <comment id="1919199" author="max.hirschhorn@10gen.com" created="Wed, 13 Jun 2018 03:53:08 +0000"  >&lt;p&gt;I discussed with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sam.kleinman&quot; class=&quot;user-hover&quot; rel=&quot;sam.kleinman&quot;&gt;sam.kleinman&lt;/a&gt; and he didn&apos;t recommend the remote syslog approach. We&apos;ll build something ourselves - it could be curator to send the logs somewhere, but there are a number of other options. My current thought is to just be copying the mongod log to the Evergreen-spawned machine as the powercycle test is going along rather than needing to signal with a blocking wait that logs have been flushed in a more persistent way.&lt;/p&gt;</comment>
                            <comment id="1913883" author="acm" created="Thu, 7 Jun 2018 16:03:36 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=max.hirschhorn&quot; class=&quot;user-hover&quot; rel=&quot;max.hirschhorn&quot;&gt;max.hirschhorn&lt;/a&gt; - I don&apos;t believe calls to the remote syslog are buffered though. As soon as you log the message, it goes over the wire. Why wouldn&apos;t that be sufficient, as long as the machine you are crashing isn&apos;t the machine running the syslog daemon?&lt;/p&gt;</comment>
                            <comment id="1899616" author="max.hirschhorn@10gen.com" created="Wed, 23 May 2018 15:33:39 +0000"  >&lt;blockquote&gt;
&lt;p&gt;Max Hirschhorn - Even a separate filesystem might flow through the same IO controller, no?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I don&apos;t know how computers work so I couldn&apos;t really say.&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;Regarding flushing the stream, can we point to some actual code to discuss? Where is the write to the stream that you would like to augment with a flush?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Using the word &quot;flush&quot; was probably misleading. What I was getting at is that the remote syslog has some queue of messages which we would want to wait to be drained before triggering the crash. I don&apos;t think there&apos;s any code I can point to because it would be some behavior of the remote syslog.&lt;/p&gt;</comment>
                            <comment id="1899605" author="acm" created="Wed, 23 May 2018 15:25:40 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=max.hirschhorn&quot; class=&quot;user-hover&quot; rel=&quot;max.hirschhorn&quot;&gt;max.hirschhorn&lt;/a&gt; - Even a separate filesystem might flow through the same IO controller, no? Regarding flushing the stream, can we point to some actual code to discuss? Where is the write to the stream that you would like to augment with a flush?&lt;/p&gt;</comment>
                            <comment id="1899564" author="max.hirschhorn@10gen.com" created="Wed, 23 May 2018 15:06:35 +0000"  >&lt;blockquote&gt;
&lt;p&gt;Here is another suggestion: what about logging to a remote syslog?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=acm&quot; class=&quot;user-hover&quot; rel=&quot;acm&quot;&gt;acm&lt;/a&gt;, that doesn&apos;t seem much different than #2 as we&apos;ll still want to attempt to flush any messages to the remote syslog before crashing the machine. Could you explain why a separate filesystem for the mongod logs is potentially insufficient for avoiding an impact on the data files and journal files?&lt;/p&gt;</comment>
                            <comment id="1899514" author="acm" created="Wed, 23 May 2018 14:36:10 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=max.hirschhorn&quot; class=&quot;user-hover&quot; rel=&quot;max.hirschhorn&quot;&gt;max.hirschhorn&lt;/a&gt; - I&apos;m inclined to agree with Bruce that any sort of D_SYNC/fsync option is likely to subvert the intention of the power cycle test, and I&apos;m not particularly excited to plumb that option in all the way down the logging subsystem. We have hopes to make the logging implementation simpler and less configurable so that it can be made &quot;immortal&quot; in ways that it currently isn&apos;t, and that is definitely going to reduce its configurability. Here is another suggestion: what about logging to a remote syslog?&lt;/p&gt;</comment>
                            <comment id="1882897" author="ian@10gen.com" created="Fri, 4 May 2018 14:08:07 +0000"  >&lt;p&gt;Assigning to Platform since they have ownership of the logging subsystem and its different output methods.&lt;/p&gt;</comment>
                            <comment id="1880392" author="jonathan.abrahams" created="Wed, 2 May 2018 15:13:50 +0000"  >&lt;p&gt;The Powercycle test already uses a &lt;tt&gt;/log&lt;/tt&gt; drive for &lt;tt&gt;logpath&lt;/tt&gt; and &lt;tt&gt;/data&lt;/tt&gt; drive for &lt;tt&gt;dbpath&lt;/tt&gt;&lt;/p&gt;</comment>
                            <comment id="1880386" author="sue.loverso" created="Wed, 2 May 2018 15:09:44 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;&apos;s concern is valid. However, option 3 or 4 also seem like the most straightforward way to solve the problem. Is there is a way to measure a weakening of the test pre/post a change? I&apos;d define weakening also as a severe reduction in the amount of work the test can do, if say, &lt;tt&gt;O_DSYNC&lt;/tt&gt; causes huge delays.&lt;/p&gt;

&lt;p&gt;Perhaps one way to mitigate some durability interactions would be to apply option 3 or 4 and set the &lt;tt&gt;logpath&lt;/tt&gt; of the mongodb log file onto a separate file system than the database directory and journal files. That would be a 3rd file system if the database and journal are already separated.&lt;/p&gt;</comment>
                            <comment id="1880128" author="bruce.lucas@10gen.com" created="Wed, 2 May 2018 12:52:26 +0000"  >&lt;p&gt;One concern is that fsyncing the log file may have the potential to weaken the test by making more than the log file durable (depending on kernel and device behavior), and also by changing the timing.&lt;/p&gt;</comment>
                            <comment id="1879917" author="max.hirschhorn@10gen.com" created="Wed, 2 May 2018 02:53:47 +0000"  >&lt;p&gt;I&apos;ve sent this to the Storage team to triage because I suspect they are in the best position to comment on how to make mongod log files durable.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>12.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25142"><![CDATA[Server Tooling & Methods]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 2 May 2018 12:52:26 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        2 years, 13 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>alexander.golin@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            2 years, 13 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>andrew.morrow@mongodb.com</customfieldvalue>
            <customfieldvalue>backlog-server-stm</customfieldvalue>
            <customfieldvalue>brooke.miller@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>ian@mongodb.com</customfieldvalue>
            <customfieldvalue>jonathan.abrahams</customfieldvalue>
            <customfieldvalue>max.hirschhorn@mongodb.com</customfieldvalue>
            <customfieldvalue>sue.loverso@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htx0lz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr8ven:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htwmuv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>