<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 09:01:22 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[JAVA-4155] Investigate intermittent failures of reactive `change-streams-resume-whitelist.json:change stream resumes after CursorNotFound`</title>
                <link>https://jira.mongodb.org/browse/JAVA-4155</link>
                <project id="10006" key="JAVA">Java Driver</project>
                    <description>&lt;p&gt;&lt;tt&gt;change-streams-resume-whitelist.json:change stream resumes after CursorNotFound&lt;/tt&gt; sometimes hangs when run via &lt;tt&gt;com.mongodb.reactivestreams.client.ChangeStreamsTest&lt;/tt&gt;. Here are a couple of examples:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;&lt;a href=&quot;https://evergreen.mongodb.com/task/mongo_java_driver_tests_jdk8_unsecure__version~latest_os~linux_topology~replicaset_auth~noauth_ssl~nossl_jdk~jdk8_test_38117f16f6df7b2e9894d7a8e92e64179e74244f_21_04_06_13_25_47#/log/ALL&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;on a replica set&lt;/a&gt; (&lt;a href=&quot;https://evergreen.mongodb.com/task_log_raw/mongo_java_driver_tests_jdk8_unsecure__version~latest_os~linux_topology~replicaset_auth~noauth_ssl~nossl_jdk~jdk8_test_38117f16f6df7b2e9894d7a8e92e64179e74244f_21_04_06_13_25_47/0?type=ALL&amp;amp;text=true&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;logs&lt;/a&gt;)&lt;/li&gt;
	&lt;li&gt;&lt;a href=&quot;https://spruce.mongodb.com/version/609a78eb32f4173cd1678d27/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC&amp;amp;variant=tests-jdk8-unsecure__version~latest_os~linux_topology~sharded-cluster_auth~noauth_ssl~nossl_jdk~jdk8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;on a sharded cluster&lt;/a&gt;&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment></environment>
        <key id="1714589">JAVA-4155</key>
            <summary>Investigate intermittent failures of reactive `change-streams-resume-whitelist.json:change stream resumes after CursorNotFound`</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="10300" iconUrl="https://jira.mongodb.org/images/icons/priorities/medium.svg">Unknown</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="valentin.kovalenko@mongodb.com">Valentin Kavalenka</assignee>
                                    <reporter username="valentin.kovalenko@mongodb.com">Valentin Kavalenka</reporter>
                        <labels>
                            <label>flaky-tests</label>
                    </labels>
                <created>Thu, 13 May 2021 13:27:48 +0000</created>
                <updated>Mon, 17 May 2021 19:57:09 +0000</updated>
                            <resolved>Mon, 17 May 2021 19:56:59 +0000</resolved>
                                                                    <component>Test Failure</component>
                                        <votes>0</votes>
                                    <watches>2</watches>
                                                                                                                <comments>
                            <comment id="3778172" author="JIRAUSER1258163" created="Mon, 17 May 2021 19:56:59 +0000"  >&lt;p&gt;Split to &lt;a href=&quot;https://jira.mongodb.org/browse/JAVA-4165&quot; title=&quot;`ChangeStreamPublisher` may lose change events when retrying `getMore`&quot; class=&quot;issue-link&quot; data-issue-key=&quot;JAVA-4165&quot;&gt;&lt;del&gt;JAVA-4165&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="3772821" author="JIRAUSER1258163" created="Thu, 13 May 2021 21:25:15 +0000"  >&lt;p&gt;Found the root cause (explained below). This bug exists at least since &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/commit/eadbc5c8d85fd3efc21271a2155a068adf4346bf&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;eadbc5c8d85fd3efc21271a2155a068adf4346bf&lt;/a&gt; and allows change streams to miss change events each time &lt;tt&gt;AsyncQueryBatchCursor.next&lt;/tt&gt; fails with a retryable error.&lt;/p&gt;
&lt;h6&gt;&lt;a name=&quot;Reproducing&quot;&gt;&lt;/a&gt;Reproducing&lt;/h6&gt;
&lt;ul&gt;
	&lt;li&gt;Pause a thread for 3 seconds as the very first action in the &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/blob/682d299fc8f36c16fca6328d96d33b6cb55a90b3/driver-core/src/main/com/mongodb/internal/operation/AsyncQueryBatchCursor.java#L245&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;method &lt;tt&gt;AsyncQueryBatchCursor.getMore&lt;/tt&gt;&lt;/a&gt;&lt;/li&gt;
	&lt;li&gt;Pause a thread for 5 seconds between &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/blob/682d299fc8f36c16fca6328d96d33b6cb55a90b3/driver-core/src/main/com/mongodb/internal/operation/AsyncChangeStreamBatchCursor.java#L84-L85&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;these two statements in &lt;tt&gt;AsyncChangeStreamBatchCursor.next&lt;/tt&gt;&lt;/a&gt;&lt;/li&gt;
	&lt;li&gt;&lt;tt&gt;change-streams-resume-whitelist.json:change stream resumes after CursorNotFound&lt;/tt&gt; has to be the only test that is run. I did not spent time figuring out why, but if other tests rung with the above changes, then even this test starts failing for a reason unrelated to the bug question. So all other change stream JSON tests have to be deleted.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;These changes make the test &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/blob/682d299fc8f36c16fca6328d96d33b6cb55a90b3/driver-core/src/test/resources/change-streams/change-streams-resume-whitelist.json#L1653&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;tt&gt;change-streams-resume-whitelist.json:change stream resumes after CursorNotFound&lt;/tt&gt;&lt;/a&gt; almost always hang when run via &lt;tt&gt;com.mongodb.reactivestreams.client.ChangeStreamsTest&lt;/tt&gt;.&lt;/p&gt;
&lt;h6&gt;&lt;a name=&quot;Explanation&quot;&gt;&lt;/a&gt;Explanation&lt;/h6&gt;

&lt;p&gt;The first change increases the probability that the &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/blob/682d299fc8f36c16fca6328d96d33b6cb55a90b3/driver-core/src/test/resources/change-streams/change-streams-resume-whitelist.json#L1679&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;tt&gt;insertOne&lt;/tt&gt;&lt;/a&gt; operation is executed before the very first &lt;tt&gt;getMore&lt;/tt&gt; (the one that fails with &lt;tt&gt;43 CursorNotFound&lt;/tt&gt; as a result of the &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/blob/682d299fc8f36c16fca6328d96d33b6cb55a90b3/driver-core/src/test/resources/change-streams/change-streams-resume-whitelist.json#L1656&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;tt&gt;configureFailPoint&lt;/tt&gt;&lt;/a&gt; command).&lt;/p&gt;

&lt;p&gt;The second change increases the probability of &lt;tt&gt;AsyncChangeStreamBatchCursor&lt;/tt&gt; not setting its &lt;tt&gt;resumeToken&lt;/tt&gt; via &lt;tt&gt;cachePostBatchResumeToken&lt;/tt&gt; before it retries the &lt;tt&gt;AsyncQueryBatchCursor.next&lt;/tt&gt; (fails as a result of the very first &lt;tt&gt;getMore&lt;/tt&gt; failing) and &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/blob/682d299fc8f36c16fca6328d96d33b6cb55a90b3/driver-core/src/main/com/mongodb/internal/operation/AsyncChangeStreamBatchCursor.java#L260&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;calls &lt;tt&gt;ChangeStreamOperation.setChangeStreamOptionsForResume&lt;/tt&gt;&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;With &lt;tt&gt;resumeToken&lt;/tt&gt; not set on retry, the second cursor created does continue listening where the first cursor (created before executing &lt;tt&gt;insertOne&lt;/tt&gt; because of this &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/blob/682d299fc8f36c16fca6328d96d33b6cb55a90b3/driver-sync/src/test/functional/com/mongodb/client/AbstractChangeStreamsTest.java#L126&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;waiting&lt;/a&gt;) stopped listening, and misses the change notification corresponding to the &lt;tt&gt;insertOne&lt;/tt&gt; operation.&lt;/p&gt;
&lt;h6&gt;&lt;a name=&quot;Fixing&quot;&gt;&lt;/a&gt;Fixing&lt;/h6&gt;

&lt;p&gt;I need to make sense of the juggling around &lt;tt&gt;resumeToken&lt;/tt&gt; in &lt;tt&gt;AsyncChangeStreamBatchCursor&lt;/tt&gt; before I can come up with a solution.&lt;/p&gt;</comment>
                            <comment id="3771506" author="JIRAUSER1258163" created="Thu, 13 May 2021 14:00:30 +0000"  >&lt;p&gt;I don&apos;t think this is due to the logging level, because I can see the following &lt;tt&gt;killCursors&lt;/tt&gt; command in the server logs (&lt;tt&gt;mongod-vq8d49p9/mongod.log&lt;/tt&gt;), that was clearly sent by a test run:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{&quot;t&quot;:{&quot;$date&quot;:&quot;2021-05-13T00:51:12.876+00:00&quot;},&quot;s&quot;:&quot;I&quot;,  &quot;c&quot;:&quot;COMMAND&quot;,  &quot;id&quot;:51803,   &quot;ctx&quot;:&quot;conn77&quot;,&quot;msg&quot;:&quot;Slow query&quot;,&quot;attr&quot;:{&quot;type&quot;:&quot;command&quot;,&quot;ns&quot;:&quot;change-stream-tests.test&quot;,&quot;command&quot;:{&quot;killCursors&quot;:&quot;test&quot;,&quot;cursors&quot;:[4715714928219243784],&quot;lsid&quot;:{&quot;id&quot;:{&quot;$uuid&quot;:&quot;386eda02-6028-4a2e-8733-b01e45e371fd&quot;},&quot;uid&quot;:{&quot;$binary&quot;:{&quot;base64&quot;:&quot;47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=&quot;,&quot;subType&quot;:&quot;0&quot;}}},&quot;$clusterTime&quot;:{&quot;clusterTime&quot;:{&quot;$timestamp&quot;:{&quot;t&quot;:1620867072,&quot;i&quot;:18}},&quot;signature&quot;:{&quot;hash&quot;:{&quot;$binary&quot;:{&quot;base64&quot;:&quot;zsigIQSVXZfUu7SupN1pqteNmW8=&quot;,&quot;subType&quot;:&quot;0&quot;}},&quot;keyId&quot;:6961570154870210581}},&quot;$client&quot;:{&quot;driver&quot;:{&quot;name&quot;:&quot;mongo-java-driver|reactive-streams&quot;,&quot;version&quot;:&quot;4.3.0-beta1-13-gddefdb0-dirty&quot;},&quot;os&quot;:{&quot;type&quot;:&quot;Linux&quot;,&quot;name&quot;:&quot;Linux&quot;,&quot;architecture&quot;:&quot;amd64&quot;,&quot;version&quot;:&quot;3.10.0-327.22.2.el7.x86_64&quot;},&quot;platform&quot;:&quot;Java/Oracle Corporation/1.8.0_162-b12&quot;,&quot;mongos&quot;:{&quot;host&quot;:&quot;ip-10-122-84-97.ec2.internal:27017&quot;,&quot;client&quot;:&quot;127.0.0.1:39384&quot;,&quot;version&quot;:&quot;4.4.3&quot;}},&quot;$configServerState&quot;:{&quot;opTime&quot;:{&quot;ts&quot;:{&quot;$timestamp&quot;:{&quot;t&quot;:1620867072,&quot;i&quot;:17}},&quot;t&quot;:1}},&quot;$db&quot;:&quot;change-stream-tests&quot;},&quot;numYields&quot;:0,&quot;reslen&quot;:345,&quot;locks&quot;:{},&quot;protocol&quot;:&quot;op_msg&quot;,&quot;durationMillis&quot;:0}}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;However, there is no cursor ID 4715714928219243784 in the corresponding &lt;span class=&quot;error&quot;&gt;&amp;#91;client logs&amp;#93;&lt;/span&gt;(&lt;a href=&quot;https://evergreen.mongodb.com/lobster/evergreen/task/mongo_java_driver_tests_jdk8_unsecure__version~4.4_os~linux_topology~sharded_cluster_auth~noauth_ssl~nossl_jdk~jdk8_test_patch_ddefdb0cc29541f82d507318ef7f94ec03efcd39_609c76c7c9ec444f5206a57c_21_05_13_00_45_59/0/task#bookmarks=0%2C2305%2C2505%2C2519%2C3846&amp;amp;shareLine=2269&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://evergreen.mongodb.com/lobster/evergreen/task/mongo_java_driver_tests_jdk8_unsecure__version~4.4_os~linux_topology~sharded_cluster_auth~noauth_ssl~nossl_jdk~jdk8_test_patch_ddefdb0cc29541f82d507318ef7f94ec03efcd39_609c76c7c9ec444f5206a57c_21_05_13_00_45_59/0/task#bookmarks=0%2C2305%2C2505%2C2519%2C3846&amp;amp;shareLine=2269&lt;/a&gt;). So the server does log what I would expect, but the attached server and client logs do not appear to be produced by the same run on Evergreen.&lt;/p&gt;

&lt;p&gt;The timestamps, on the other hand, match (the following is a client log entry corresponding to the only &lt;tt&gt;killCursors&lt;/tt&gt; command issued by the test):&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[2021/05/13 00:51:12.930]     00:51:12.872 [Thread-9] DEBUG org.mongodb.driver.protocol.command - Sending command &apos;{&quot;killCursors&quot;: &quot;test&quot;, &quot;cursors&quot;: [3414173270393235723], &quot;$db&quot;: &quot;change-stream-tests&quot;, &quot;$clusterTime&quot;: {&quot;clusterTime&quot;: {&quot;$timestamp&quot;: {&quot;t&quot;: 1620867072, &quot;i&quot;: 17}}, &quot;signature&quot;: {&quot;hash&quot;: {&quot;$binary&quot;: {&quot;base64&quot;: &quot;AAAAAAAAAAAAAAAAAAAAAAAAAAA=&quot;, &quot;subType&quot;: &quot;00&quot;}}, &quot;keyId&quot;: 0}}, &quot;lsid&quot;: {&quot;id&quot;: {&quot;$binary&quot;: {&quot;base64&quot;: &quot;OG7aAmAoSi6HM7AeReNx/Q==&quot;, &quot;subType&quot;: &quot;04&quot;}}}}&apos; with request id 22 to database change-stream-tests on connection [connectionId{localValue:6, serverValue:30}] to server localhost:27017&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="3771485" author="jeff.yemin" created="Thu, 13 May 2021 13:48:04 +0000"  >&lt;p&gt;It&apos;s possible that the log verbosity configured for the server is not high enough to show what you&apos;re looking for.&lt;/p&gt;</comment>
                            <comment id="3771448" author="JIRAUSER1258163" created="Thu, 13 May 2021 13:33:22 +0000"  >&lt;p&gt;My best reproduction with &lt;tt&gt;TRACE&lt;/tt&gt; logging enabled and with additional &lt;tt&gt;VAKO&lt;/tt&gt; logs is &lt;a href=&quot;https://spruce.mongodb.com/task/mongo_java_driver_tests_jdk8_unsecure__version~4.4_os~linux_topology~sharded_cluster_auth~noauth_ssl~nossl_jdk~jdk8_test_patch_ddefdb0cc29541f82d507318ef7f94ec03efcd39_609c76c7c9ec444f5206a57c_21_05_13_00_45_59/logs?execution=0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;on a sharded cluster&lt;/a&gt; (&lt;a href=&quot;https://evergreen.mongodb.com/lobster/evergreen/task/mongo_java_driver_tests_jdk8_unsecure__version~4.4_os~linux_topology~sharded_cluster_auth~noauth_ssl~nossl_jdk~jdk8_test_patch_ddefdb0cc29541f82d507318ef7f94ec03efcd39_609c76c7c9ec444f5206a57c_21_05_13_00_45_59/0/task#bookmarks=0%2C2305%2C2505%2C2519%2C3846&amp;amp;shareLine=2269&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;logs&lt;/a&gt;). As far as I can tell, in situations when the test hangs, it hangs by infinitely requesting &lt;tt&gt;getMore&lt;/tt&gt; because the change stream never receives a change document corresponding to the &lt;tt&gt;insertOne&lt;/tt&gt; operation of the test. So the test runner is blocked in the &lt;a href=&quot;https://github.com/mongodb/mongo-java-driver/blob/2c926dfdb349fa0ae67613a4b6ae9019c3ea2397/driver-sync/src/test/functional/com/mongodb/client/AbstractChangeStreamsTest.java#L141&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;tt&gt;cursor.next&lt;/tt&gt;&lt;/a&gt; until Evergreen terminates the test run.&lt;/p&gt;

&lt;p&gt;I wanted to see if server logs could clarify whether or not the server ever replied to &lt;tt&gt;getMore&lt;/tt&gt; with a change stream document, but the logs attached by the Evergreen do not seem to correspond to the test run: neither the cursor ID 3414173270393235723 nor the &lt;tt&gt;configureFailPoint&lt;/tt&gt; command are present in the logs.&lt;/p&gt;

&lt;p&gt;I will try to reproduce on a replica set, maybe with a different topology Evergreen attaches relevant logs, or maybe I am misinterpreting the lack of cursor ID / &lt;tt&gt;configureFailPoint&lt;/tt&gt; in the sharded cluster logs.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="1601857">JAVA-3970</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10620">
                    <name>Issue split</name>
                                            <outwardlinks description="split to">
                                        <issuelink>
            <issuekey id="1719430">JAVA-4165</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10257" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Documentation Changes</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="11861"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>JAVA-3971</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hyvbnj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            </customfields>
    </item>
</channel>
</rss>