<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 08:38:36 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[GODRIVER-2438] Pool &quot;closeConnection&quot; method cause panics on removePerishedConns()</title>
                <link>https://jira.mongodb.org/browse/GODRIVER-2438</link>
                <project id="14289" key="GODRIVER">Go Driver</project>
                    <description>&lt;h4&gt;&lt;a name=&quot;Summary&quot;&gt;&lt;/a&gt;Summary&lt;/h4&gt;

&lt;p&gt;&lt;em&gt;Sometimes panics occurs and application is restarted.&lt;/em&gt;&lt;/p&gt;
&lt;h4&gt;&lt;a name=&quot;Driverversions%3E1.8.4%2C1.9.1&quot;&gt;&lt;/a&gt;Driver versions -&amp;gt; 1.8.4, 1.9.1&lt;/h4&gt;
&lt;h4&gt;&lt;a name=&quot;HowtoReproduce&quot;&gt;&lt;/a&gt;How to Reproduce&lt;/h4&gt;
&lt;h4&gt;&lt;a name=&quot;NoclearWTR%2Cit%27shappeningfromtimetotimeonproductiondeployments&quot;&gt;&lt;/a&gt;&lt;em&gt;No clear WTR, it&apos;s happening from time to time on production deployments&lt;/em&gt;&lt;/h4&gt;

&lt;p&gt;Additional Background&lt;/p&gt;

&lt;p&gt;&lt;em&gt;Stack trace:&lt;/em&gt;&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/app/vendor/&lt;a href=&quot;http://go.mongodb.org/mongo-driver/x/mongo/driver/topology/pool.go:772&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;ins&gt;go.mongodb.org/mongo-driver/x/mongo/driver/topology/pool.go:772&lt;/ins&gt;&lt;/a&gt; +0x198&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;created by &lt;a href=&quot;http://go.mongodb.org/mongo-driver/x/mongo/driver/topology&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;ins&gt;go.mongodb.org/mongo-driver/x/mongo/driver/topology&lt;/ins&gt;&lt;/a&gt;.(*pool).removePerishedConns&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/app/vendor/&lt;a href=&quot;http://go.mongodb.org/mongo-driver/x/mongo/driver/topology/pool.go:773&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;ins&gt;go.mongodb.org/mongo-driver/x/mongo/driver/topology/pool.go:773&lt;/ins&gt;&lt;/a&gt; +0x25&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;&lt;a href=&quot;http://go.mongodb.org/mongo-driver/x/mongo/driver/topology&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;ins&gt;go.mongodb.org/mongo-driver/x/mongo/driver/topology&lt;/ins&gt;&lt;/a&gt;.(*pool).removePerishedConns.func1()&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/app/vendor/&lt;a href=&quot;http://go.mongodb.org/mongo-driver/x/mongo/driver/topology/pool.go:400&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;ins&gt;go.mongodb.org/mongo-driver/x/mongo/driver/topology/pool.go:400&lt;/ins&gt;&lt;/a&gt; +0x75&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;&lt;a href=&quot;http://go.mongodb.org/mongo-driver/x/mongo/driver/topology&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;ins&gt;go.mongodb.org/mongo-driver/x/mongo/driver/topology&lt;/ins&gt;&lt;/a&gt;.(*pool).closeConnection(0xc000714a80?, 0xc0004db400)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/app/vendor/&lt;a href=&quot;http://go.mongodb.org/mongo-driver/x/mongo/driver/topology/connection.go:499&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;ins&gt;go.mongodb.org/mongo-driver/x/mongo/driver/topology/connection.go:499&lt;/ins&gt;&lt;/a&gt; +0x43&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;&lt;a href=&quot;http://go.mongodb.org/mongo-driver/x/mongo/driver/topology&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;ins&gt;go.mongodb.org/mongo-driver/x/mongo/driver/topology&lt;/ins&gt;&lt;/a&gt;.(*connection).close(0xc0004db400?)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/usr/local/go/src/crypto/tls/conn.go:1338 +0x69&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;crypto/tls.(*Conn).Close(0xc00080d701?)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/usr/local/go/src/crypto/tls/conn.go:1368 +0xdb&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;crypto/tls.(*Conn).closeNotify(0xc00162ea80)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/usr/local/go/src/crypto/tls/conn.go:151&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;crypto/tls.(*Conn).SetWriteDeadline(...)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/usr/local/go/src/net/net.go:261 +0x48&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;net.(*conn).SetWriteDeadline(0xc00000ed10, {0x51b5b106590?, 0x1596c00?, 0x1596c00?})&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/usr/local/go/src/net/fd_posix.go:146&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;net.(*netFD).SetWriteDeadline(...)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/usr/local/go/src/internal/poll/fd_poll_runtime.go:142&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;internal/poll.(*FD).SetWriteDeadline(...)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/usr/local/go/src/internal/poll/fd_poll_runtime.go:160 +0x1be&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;internal/poll.setDeadlineImpl(0xc0022c38c0, {0x2?, 0x0?, 0x1596c00?}, 0x4c17b6?)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;/usr/local/go/src/runtime/netpoll.go:324 +0x3e&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;internal/poll.runtime_pollSetDeadline(0x2, 0x12a05f0f3, 0x77)&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;goroutine 223004 &lt;span class=&quot;error&quot;&gt;&amp;#91;running&amp;#93;&lt;/span&gt;:&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;font color=&quot;#000000&quot;&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;signal SIGSEGV: segmentation violation code=0x1 addr=0x2 pc=0x46365e&amp;#93;&lt;/span&gt;&lt;/font&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;Solution:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://github.com/mongodb/mongo-go-driver/pull/966/files&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Handle panics inside closeConnection function&lt;/a&gt; - Pull request&lt;/p&gt;</description>
                <environment></environment>
        <key id="2056774">GODRIVER-2438</key>
            <summary>Pool &quot;closeConnection&quot; method cause panics on removePerishedConns()</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13203">Gone away</resolution>
                                        <assignee username="matt.dale@mongodb.com">Matt Dale</assignee>
                                    <reporter username="andrii.hrachov@deliveryhero.com">Andrii Hrachov</reporter>
                        <labels>
                    </labels>
                <created>Fri, 27 May 2022 15:26:11 +0000</created>
                <updated>Tue, 23 Jan 2024 00:25:44 +0000</updated>
                            <resolved>Wed, 30 Nov 2022 12:00:38 +0000</resolved>
                                    <version>1.8.4</version>
                    <version>1.9.1</version>
                                                                        <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="5018224" author="dbeng-pm-bot" created="Wed, 30 Nov 2022 12:00:39 +0000"  >&lt;p&gt;There hasn&apos;t been any recent activity on this ticket, so we&apos;re resolving it. Thanks for reaching out! Please feel free to comment on this if you&apos;re able to provide more information.&lt;/p&gt;</comment>
                            <comment id="4985744" author="JIRAUSER1259527" created="Wed, 16 Nov 2022 05:50:28 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=andrii.hrachov%40deliveryhero.com&quot; class=&quot;user-hover&quot; rel=&quot;andrii.hrachov@deliveryhero.com&quot;&gt;andrii.hrachov@deliveryhero.com&lt;/a&gt;&#160;after further discussion with the team, we decided to not accept the proposed PR that catches panics when closing connections via a &quot;defer recover&quot; block. &quot;Defer recover&quot; blocks are difficult to remove once added, especially if we can&apos;t reproduce the panic scenario.&lt;/p&gt;

&lt;p&gt;Do you still encounter panics when closing connections?&lt;/p&gt;</comment>
                            <comment id="4649672" author="JIRAUSER1269379" created="Thu, 30 Jun 2022 06:11:39 +0000"  >&lt;p&gt;Thanks a lot, Matt!&lt;/p&gt;

&lt;p&gt;Should open bug ticket to golang now &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/biggrin.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="4649101" author="JIRAUSER1259527" created="Wed, 29 Jun 2022 20:58:39 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=andrii.hrachov%40deliveryhero.com&quot; class=&quot;user-hover&quot; rel=&quot;andrii.hrachov@deliveryhero.com&quot;&gt;andrii.hrachov@deliveryhero.com&lt;/a&gt; I haven&apos;t been able to reproduce the exact panic condition you described, but it makes sense that if there is any possibility that closing a &lt;tt&gt;net.Conn&lt;/tt&gt; can panic, we should recover that panic and return it as an error. I&apos;ve opened &lt;a href=&quot;https://github.com/mongodb/mongo-go-driver/pull/1007&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;PR 1007&lt;/a&gt; that adds a &lt;tt&gt;defer recover&lt;/tt&gt; block to &lt;tt&gt;connection.close()&lt;/tt&gt;. I&apos;ve also created &lt;a href=&quot;https://github.com/mongodb/mongo-go-driver/pull/994&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;PR 994&lt;/a&gt; that should prevent the RTT monitor from getting stuck running an operation indefinitely.&lt;/p&gt;

&lt;p&gt;The original stack trace that you posted suggests the panic is actually coming from somewhere in Go&apos;s &quot;netpoller&quot;, which could point to a bug in Go&apos;s network connection code or memory corruption caused by data races. It may be helpful to build and run your application with the data race detector enabled (&lt;tt&gt;go build -race&lt;/tt&gt;) to check for memory corruption. There are also a few similar open issues on the &lt;a href=&quot;https://github.com/golang/go&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;github.com/golang/go&lt;/a&gt; repository, including one about &lt;a href=&quot;https://github.com/golang/go/issues/51113&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;segfault panics from Go apps running in Docker&lt;/a&gt;. You could consider opening a new issue on the Go project repo with a description of your problem, optionally linking to this ticket. If you do open an issue on the Go project, please add a link to that issue here as well!&lt;/p&gt;

&lt;p&gt;The two linked PRs are scheduled for release with Go Driver v1.10.0, which will be released in late July.&lt;/p&gt;</comment>
                            <comment id="4619705" author="JIRAUSER1259527" created="Thu, 16 Jun 2022 04:19:39 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=andrii.hrachov%40deliveryhero.com&quot; class=&quot;user-hover&quot; rel=&quot;andrii.hrachov@deliveryhero.com&quot;&gt;andrii.hrachov@deliveryhero.com&lt;/a&gt; that&apos;s really helpful information! I&apos;m going to try to create a reproducible example of the panic conditions you described. I&apos;m also evaluating our use of &quot;defer recover&quot; blocks in the Go Driver to see if there are more or different places they should be added. I&apos;ll respond again when I have more information.&lt;/p&gt;</comment>
                            <comment id="4585856" author="JIRAUSER1269379" created="Wed, 1 Jun 2022 08:44:22 +0000"  >&lt;p&gt;So what i see:&lt;/p&gt;

&lt;p&gt;When connections are closed above application level (os, mesh etc), mongodb driver panicks when trying to close connection or measure RTT&lt;/p&gt;</comment>
                            <comment id="4585851" author="JIRAUSER1269379" created="Wed, 1 Jun 2022 08:35:34 +0000"  >&lt;p&gt;1) 1.9.1&lt;br/&gt;
2) 4.2&lt;br/&gt;
3)&#160;&lt;/p&gt;

&lt;p&gt;```&lt;/p&gt;

&lt;p&gt;clientOptions := options.&lt;br/&gt;
Client().&lt;br/&gt;
SetRegistry(m.registry).&lt;br/&gt;
SetWriteConcern(writeconcern.New(writeconcern.WMajority())).&lt;br/&gt;
SetReadPreference(readPreference.Primary()).&lt;br/&gt;
SetDisableOCSPEndpointCheck(true).&lt;br/&gt;
SetServerSelectionTimeout(120 * time.Second).&lt;br/&gt;
SetMaxPoolSize(100).&lt;br/&gt;
SetMinPoolSize(20).&lt;br/&gt;
SetMaxConnIdleTime(60 * time.Second)```&lt;/p&gt;

&lt;p&gt;4) 1.18&lt;/p&gt;

&lt;p&gt;5) I already lost the stack trace but the problem was in RTT monitor, my fix was;&lt;/p&gt;

&lt;p&gt;```&lt;br/&gt;
func (r *rttMonitor) runHello(conn *connection) *connection {&lt;br/&gt;
defer func() {&lt;br/&gt;
if r := recover(); r != nil {&lt;br/&gt;
if conn != nil &amp;amp;&amp;amp; !conn.closed()&lt;/p&gt;

{ _ = conn.close() }

&lt;p&gt;}&lt;br/&gt;
}()&lt;br/&gt;
```&lt;/p&gt;

&lt;p&gt;The exact line was panicking:&lt;/p&gt;

&lt;p&gt;```&lt;br/&gt;
err := r.cfg.createOperationFn(initConnection&lt;/p&gt;

{conn}

&lt;p&gt;).Execute(r.ctx)&lt;br/&gt;
```&lt;/p&gt;

&lt;p&gt;&lt;b&gt;More info investigated:&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;Actually with forking driver and putting recovers in connection pool background operations helped, but there were 3 more contribution factors:&lt;/p&gt;

&lt;p&gt;1) Some requests were dangling forever (no context cancellation), so when the fd poll timed out - driver panicked, because the connection was already gone on system side. Introducing app timeout helped to mitigate some problems but not all of them&lt;br/&gt;
2) We were using istio for everything and it gave sporadic tls connection failures, this also caused connections to be tampered and also resulted to panic on the driver side (when it tries to measure RTT or close the connection)&lt;br/&gt;
3) also connection pool idle timeout was not set, thus it was resulting to connections close by OS, and driver also panicking when trying to reap them&lt;/p&gt;</comment>
                            <comment id="4585052" author="JIRAUSER1259527" created="Tue, 31 May 2022 22:43:40 +0000"  >&lt;p&gt;Hey &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=andrii.hrachov%40deliveryhero.com&quot; class=&quot;user-hover&quot; rel=&quot;andrii.hrachov@deliveryhero.com&quot;&gt;andrii.hrachov@deliveryhero.com&lt;/a&gt; thanks for the ticket! I have a few questions to help me identify the issue:&lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;What version of the Go Driver were you using when you captured the stack trace you provided?&lt;/li&gt;
	&lt;li&gt;What version of the MongoDB server are you using?&lt;/li&gt;
	&lt;li&gt;Can you post a sample of the MongoDB &lt;tt&gt;Client&lt;/tt&gt; configuration you&apos;re using?&lt;/li&gt;
	&lt;li&gt;What version of Go are you using?&lt;/li&gt;
	&lt;li&gt;You mention you also get a panic from &lt;tt&gt;runHello&lt;/tt&gt;. Can you post a stack trace of that panic as well?&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;Thanks,&lt;br/&gt;
Matt&lt;/p&gt;</comment>
                            <comment id="4579906" author="JIRAUSER1269379" created="Sun, 29 May 2022 07:18:47 +0000"  >&lt;p&gt;Actually I forked the driver to fix that, and realised that another place is also need attention -&lt;/p&gt;

&lt;p&gt;runHello function. So actually the ticket can be unified- &lt;b&gt;&quot;Handle panics in connection pool background operations&quot;&lt;/b&gt;&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="2074688">GODRIVER-2467</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="2051824">GODRIVER-2432</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="2071783">GODRIVER-2464</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i0fgnc:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            </customfields>
    </item>
</channel>
</rss>