<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:09:12 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-45587] change the format of SIGUSR2 stack dumps</title>
                <link>https://jira.mongodb.org/browse/SERVER-45587</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;All threads&apos; stack traces are dumped together in one large log statement as a JSON document.&lt;/p&gt;

&lt;p&gt;The request is to downgrade these in a way, breaking the large json document into a sequence of separate log statements, one per thread, reusing the crash-style stacktrace format (with some changes to obscure any raw addresses). These should be bracketed by some metadata log statements indicating beginning and&#160; end of the stack trace set.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1097056">SERVER-45587</key>
            <summary>change the format of SIGUSR2 stack dumps</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="billy.donahue@mongodb.com">Billy Donahue</assignee>
                                    <reporter username="billy.donahue@mongodb.com">Billy Donahue</reporter>
                        <labels>
                    </labels>
                <created>Wed, 15 Jan 2020 17:06:41 +0000</created>
                <updated>Sun, 29 Oct 2023 22:13:14 +0000</updated>
                            <resolved>Thu, 30 Jan 2020 17:44:40 +0000</resolved>
                                                    <fixVersion>4.3.4</fixVersion>
                                    <component>Internal Code</component>
                                        <votes>0</votes>
                                    <watches>5</watches>
                                                                                                                <comments>
                            <comment id="2773518" author="xgen-internal-githook" created="Thu, 30 Jan 2020 18:51:01 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;BillyDonahue&apos;, &apos;name&apos;: &apos;Billy Donahue&apos;, &apos;email&apos;: &apos;billy.donahue@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45587&quot; title=&quot;change the format of SIGUSR2 stack dumps&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45587&quot;&gt;&lt;del&gt;SERVER-45587&lt;/del&gt;&lt;/a&gt; remove unused variable&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/e6468fe7daf0da3cd82521202ca32d6781242250&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/e6468fe7daf0da3cd82521202ca32d6781242250&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2773330" author="billy.donahue" created="Thu, 30 Jan 2020 17:44:40 +0000"  >&lt;p&gt;Fixed by &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45068&quot; title=&quot;update mongosymb.py to handle multi-thread stack traces&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45068&quot;&gt;&lt;del&gt;SERVER-45068&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2773321" author="xgen-internal-githook" created="Thu, 30 Jan 2020 17:42:13 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;BillyDonahue&apos;, &apos;name&apos;: &apos;Billy Donahue&apos;, &apos;email&apos;: &apos;billy.donahue@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45587&quot; title=&quot;change the format of SIGUSR2 stack dumps&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45587&quot;&gt;&lt;del&gt;SERVER-45587&lt;/del&gt;&lt;/a&gt; SIGUSR2 one thread per log line&lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;add mongosymb_multithread.py&lt;/li&gt;
	&lt;li&gt;mongosymb.py argparse and slight refactor for reuse.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt; create mode 100755 buildscripts/mongosymb_multithread.py&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/bc68d8424d15a9f3eeab35730b0043459e289ad9&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/bc68d8424d15a9f3eeab35730b0043459e289ad9&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2763649" author="billy.donahue" created="Fri, 24 Jan 2020 23:04:20 +0000"  >&lt;p&gt;commit queue&lt;/p&gt;</comment>
                            <comment id="2762256" author="bruce.lucas@10gen.com" created="Fri, 24 Jan 2020 16:42:43 +0000"  >&lt;p&gt;It&apos;s good to hear that we can disable truncation as needed, but I would still be worried about the practical impact of logging this as a single JSON line that might end up being multiple MB (say 1k per stack trace times 10s of thousands of threads). A quick test with emacs indicates that it can handle such a long line, but performance moving the cursor around at points after the line gets very slow. Other log viewing tools might have other problems.&lt;/p&gt;</comment>
                            <comment id="2762216" author="henrik.edin" created="Fri, 24 Jan 2020 16:30:58 +0000"  >&lt;p&gt;I got the code review for this and I realize that I&apos;m a little late to the party. But I feel like the format in the logs is missing a great opportunity to take more advantage of Structured logging instead of separating the output in multiple log statements like the current implementation.&lt;/p&gt;

&lt;p&gt;A more full use of structured logging could have the output to be something like this:&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&quot;id&quot;: X,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&quot;attr&quot;: {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  &quot;prologue&quot;: {...} // everything needed to symbolize&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  &quot;threads&quot;: [{...}, {...}, ..., {...}] // JSON array of each thread containing a subobject for everything related to each thread&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;It would result in a very large object but this log statement would have truncation turned off. But we&apos;d get a single statement with a single ID to look up when parsing.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Anyway, just wanted to point that out.&lt;/p&gt;</comment>
                            <comment id="2740578" author="bruce.lucas@10gen.com" created="Sat, 18 Jan 2020 15:11:36 +0000"  >&lt;p&gt;That looks great. Thanks for doing all this - I think this will be a big help in diagnosing some difficult problems in customer situations.&lt;/p&gt;</comment>
                            <comment id="2740264" author="billy.donahue" created="Fri, 17 Jan 2020 22:35:22 +0000"  >&lt;p&gt;sample output from&#160;&lt;a href=&quot;https://mongodbcr.appspot.com/544970038/#ps533200074&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://mongodbcr.appspot.com/544970038/#ps533200074&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/243505/243505_mongod.symb.txt&quot; title=&quot;mongod.symb.txt attached to SERVER-45587&quot;&gt;mongod.symb.txt&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="2740256" author="billy.donahue" created="Fri, 17 Jan 2020 22:30:59 +0000"  >&lt;p&gt;&lt;a href=&quot;https://mongodbcr.appspot.com/544970038/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://mongodbcr.appspot.com/544970038/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;super early sneak preview&lt;/p&gt;

&lt;p&gt;Made a new tool: mongosymb_multithread.py that calls mongosymb.py.&lt;br/&gt;
So I don&apos;t touch mongosymb.py.&lt;/p&gt;</comment>
                            <comment id="2740032" author="bruce.lucas@10gen.com" created="Fri, 17 Jan 2020 20:36:01 +0000"  >&lt;p&gt;At a quick look I think that looks good.&lt;/p&gt;

&lt;p&gt;(I think the wrapping in the log line, i.e. &quot;msg&quot;:&quot;{rec}&quot;&#160;and &quot;rec&quot;:{...} aren&apos;t great, but I think that&apos;s probably a discussion for the structured logging project.)&lt;/p&gt;</comment>
                            <comment id="2739990" author="billy.donahue" created="Fri, 17 Jan 2020 20:06:58 +0000"  >&lt;p&gt;&lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/243469/243469_mongod.out.json&quot; title=&quot;mongod.out.json attached to SERVER-45587&quot;&gt;mongod.out.json&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt;&#160;is the json structured log after the proposed refactor. WDYT?&lt;/p&gt;</comment>
                            <comment id="2739641" author="billy.donahue" created="Fri, 17 Jan 2020 18:27:21 +0000"  >&lt;p&gt;Yes, &quot;useless&quot; was overstated. Withdrawn! &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&#160; Only the &quot;b&quot; and &quot;o&quot; fields would be useless. Sometimes you don&apos;t get an &quot;s&quot;.&lt;/p&gt;

&lt;p&gt;Yes, we can emit the processInfo first. The &quot;somap&quot; is trimmed to only include libraries that are actually&#160;IN the emitted backtraces. So I&apos;ll have to reorg a bit to make this two-pass. It will work fine.&lt;/p&gt;</comment>
                            <comment id="2739587" author="bruce.lucas@10gen.com" created="Fri, 17 Jan 2020 17:59:44 +0000"  >&lt;p&gt;Good point. Does that info need to go at the end - can&apos;t it be part of the prolog?&lt;/p&gt;

&lt;p&gt;Also I think &quot;useless&quot; is an overstatement - depending on the kind of problem you&apos;re tracking down, I&apos;ve often found that just the symbolization provided by the &quot;s&quot; fields is enough to understand the problem. But yes would be better to ensure that we can do the full post-hoc symbolization.&lt;/p&gt;</comment>
                            <comment id="2739569" author="billy.donahue" created="Fri, 17 Jan 2020 17:50:15 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;, one concern I have with the proposal is that the backtrace lines are useless until you get to the epilogue.&lt;br/&gt;
 If something stops the epilogue from being printed out in full, or if it just isn&apos;t captured by the support customer, we can&apos;t use any of the other lines. In the very wasteful version in my current PoC (attached), the lines are at least usable one-at-a-time.&lt;/p&gt;</comment>
                            <comment id="2739562" author="bruce.lucas@10gen.com" created="Fri, 17 Jan 2020 17:46:51 +0000"  >&lt;p&gt;I think that sounds good.&lt;/p&gt;</comment>
                            <comment id="2739559" author="billy.donahue" created="Fri, 17 Jan 2020 17:45:29 +0000"  >&lt;p&gt;Yeah, I just want to make sure that&apos;s acceptable before proceeding.&lt;br/&gt;
 So the proposal now:&lt;/p&gt;

&lt;p&gt;First, there will be a prologue line indicating the start of a stacktrace group.&lt;br/&gt;
 It will contain a json array with tids of all the threads we missed, if there were any.&lt;/p&gt;

&lt;p&gt;For each thread, a log line of one json object containing a &quot;name&quot;, &quot;tid&quot; and &quot;backtrace&quot; array.&lt;br/&gt;
 Each element of a backtrace array has &quot;b&quot;, &quot;o&quot; fields, and maybe more hints like &quot;s&quot; and &quot;s+&quot; if known (sym and symOffset).&#160;&lt;/p&gt;

&lt;p&gt;After all threads, there will be an epilogue log line indicating the end of a stacktrace group. This is a json object containing the &quot;processInfo&quot; field.&lt;/p&gt;

&lt;p&gt;The &quot;b&quot; field in each frame is a made-up key indicating a library that will appear in the &quot;processInfo&quot; field of the epilogue line. It is not an address. This keeps addresses from being disclosed for a running server.&lt;/p&gt;

&lt;p&gt;Mongosymb.py will be modified to accept the entire block of log lines from prologue to epilogue, transforming its input into an output with the mongosymb.py enhanced dwarf info we&apos;re used to getting, but otherwise not losing any data.&lt;/p&gt;</comment>
                            <comment id="2739526" author="bruce.lucas@10gen.com" created="Fri, 17 Jan 2020 17:30:19 +0000"  >&lt;p&gt;Can we just dump the shared metadata once, and adjust mongosymb.py to understand (or write a new wrapper script)?&lt;/p&gt;</comment>
                            <comment id="2739522" author="billy.donahue" created="Fri, 17 Jan 2020 17:27:02 +0000"  >&lt;p&gt;As a proof-of-concept,&#160;&lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/243365/243365_mongod.out&quot; title=&quot;mongod.out attached to SERVER-45587&quot;&gt;mongod.out&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt;&#160;is an example of what we&apos;d be producing in response to SIGUSR2.&lt;/p&gt;

&lt;p&gt;Each line works with mongosymb.py after stripping the&lt;/p&gt;

&lt;p&gt;&#160; &#160; &quot;2020-01-17T12:18:03.825-0500 I COMMAND &lt;span class=&quot;error&quot;&gt;&amp;#91;SignalHandler&amp;#93;&lt;/span&gt;...&quot;&lt;/p&gt;

&lt;p&gt;line prefixes.&lt;/p&gt;

&lt;p&gt;In a technical sense, this is a huge downgrade in efficiency.&lt;/p&gt;

&lt;p&gt;Compare to&#160;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/239195/test.out.txt&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;original log&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Each thread dump has the FULL &quot;.processInfo&quot; field, including mongod version and &quot;.uname&quot;, etc.&lt;br/&gt;
 We&apos;re also dumping out the library metadata for each library on each thread in the &quot;.somap&quot;.&lt;/p&gt;</comment>
                            <comment id="2732498" author="bruce.lucas@10gen.com" created="Wed, 15 Jan 2020 19:34:41 +0000"  >&lt;p&gt;This seems important, as in production there can be tens of thousands of threads, making the log line unworkably large (possibly even triggering truncation?)&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                        <issuelink>
            <issuekey id="1048497">SERVER-45068</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="243365" name="mongod.out" size="58694" author="billy.donahue@mongodb.com" created="Fri, 17 Jan 2020 17:22:04 +0000"/>
                            <attachment id="243469" name="mongod.out.json" size="35459" author="billy.donahue@mongodb.com" created="Fri, 17 Jan 2020 20:06:26 +0000"/>
                            <attachment id="243505" name="mongod.symb.txt" size="83869" author="billy.donahue@mongodb.com" created="Fri, 17 Jan 2020 22:35:20 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>19.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 15 Jan 2020 19:34:41 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 1 week, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1323</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 1 week, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>billy.donahue@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>henrik.edin@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwi8gv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hw6awf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3511">Dev Tools 2020-01-27</customfieldvalue>
    <customfieldvalue id="3512">Dev Tools 2020-02-10</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwhuq7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>