<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:52:05 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-19794] Fatal Assertion 16360: duplicate key error during replication</title>
                <link>https://jira.mongodb.org/browse/SERVER-19794</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:37:49.530+0000 E REPL     [repl writer worker 9] writer worker caught exception:  :: caused by :: 11000 E11000 duplicate key error collection: talk-vsession.visitorSessions index: _id_ dup key: { : ObjectId(&apos;55c05dbdd896ff11c05bf548&apos;) } on: { ts: Timestamp 1438670269000|632, h: -3999426385275226819, v: 2, op: &quot;u&quot;, ns: &quot;talk-vsession.visitorSessions&quot;, o2: { _id: ObjectId(&apos;55c05dbdd896ff11c05bf548&apos;) }, o: { $set: { sm: &quot;e56d8d2df5c4e6761a36b7e88efe0681853a6f54&quot;, uo: new Date(1438670269476) } } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:37:49.530+0000 I -        [repl writer worker 9] Fatal Assertion 16360&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:37:49.530+0000 I -        [repl writer worker 9] &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;***aborting after fassert() failure&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="224388">SERVER-19794</key>
            <summary>Fatal Assertion 16360: duplicate key error during replication</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="milkie@mongodb.com">Eric Milkie</assignee>
                                    <reporter username="shankar.k">Shankar Karuppiah</reporter>
                        <labels>
                    </labels>
                <created>Thu, 6 Aug 2015 09:36:53 +0000</created>
                <updated>Wed, 31 May 2017 15:29:25 +0000</updated>
                            <resolved>Sat, 3 Oct 2015 12:51:46 +0000</resolved>
                                    <version>3.0.5</version>
                                                    <component>Index Maintenance</component>
                    <component>Replication</component>
                                        <votes>1</votes>
                                    <watches>21</watches>
                                                                                                                <comments>
                            <comment id="1258542" author="ramon.fernandez" created="Sat, 7 May 2016 18:07:05 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=saisrinivase%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;saisrinivase@gmail.com&quot;&gt;saisrinivase@gmail.com&lt;/a&gt;, for MongoDB-related support discussion please post on the &lt;a href=&quot;http://groups.google.com/group/mongodb-user&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;mongodb-user group&lt;/a&gt; or &lt;a href=&quot;http://stackoverflow.com/questions/tagged/mongodb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Stack Overflow with the &lt;tt&gt;mongodb&lt;/tt&gt; tag&lt;/a&gt;, where your question will reach a larger audience. A question like this involving more discussion would be best posted on the &lt;a href=&quot;http://groups.google.com/group/mongodb-user&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;mongodb-user group&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1258445" author="saisrinivase@gmail.com" created="Sat, 7 May 2016 05:42:27 +0000"  >&lt;p&gt;Hi Sachin,&lt;/p&gt;

&lt;p&gt;Sorry for the delay in response .&lt;/p&gt;

&lt;p&gt;We didn&apos;t upgraded or either down graded the version ,I have restarted re&lt;br/&gt;
sync and now it became in sync with primary.&lt;/p&gt;

&lt;p&gt;But the major concern is some times unexpectedly mongo services are going&lt;br/&gt;
down and we are starting manually in sharding.&lt;/p&gt;

&lt;p&gt;When I looked in the error log I  didn&apos;t see any errors and I am not sure&lt;br/&gt;
how I can really resolve  this issue.&lt;/p&gt;

&lt;p&gt;Any clue can you help me guys in this.&lt;/p&gt;
</comment>
                            <comment id="1257337" author="sachinrohdia@gmail.com" created="Fri, 6 May 2016 06:13:28 +0000"  >&lt;p&gt;Hello Ramon ,&lt;/p&gt;

&lt;p&gt; Thanks for your response , But I wanted to know if anyone who did the upgrade got the issue resolved 100% ., if with the newer version i get the same issue , then there is no point to upgrade .&lt;/p&gt;


&lt;p&gt;I am facing only this issue rest mongo is doing good for me .. So wanted to be 100% sure  before upgrade .&lt;/p&gt;

&lt;p&gt;Thanks &lt;br/&gt;
Sachin Kumar&lt;/p&gt;</comment>
                            <comment id="1257010" author="ramon.fernandez" created="Thu, 5 May 2016 20:14:37 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sachinrohdia%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;sachinrohdia@gmail.com&quot;&gt;sachinrohdia@gmail.com&lt;/a&gt;, if you&apos;re seeing the behavior described in this ticket it could be due to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-21275&quot; title=&quot;Document not found due to WT commit visibility issue&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-21275&quot;&gt;&lt;del&gt;SERVER-21275&lt;/del&gt;&lt;/a&gt;, so I&apos;d recommend you upgrade to the latest version (MongoDB 3.2.6 at the time of this writing) and open a new ticket if the issue persists.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1256598" author="sachinrohdia@gmail.com" created="Thu, 5 May 2016 15:55:34 +0000"  >&lt;p&gt;Hello Sai ,&lt;/p&gt;

&lt;p&gt; Is upgrade to 3.0.11 , resolved this issue ?&lt;/p&gt;</comment>
                            <comment id="1234597" author="saisrinivase@gmail.com" created="Wed, 13 Apr 2016 14:20:58 +0000"  >&lt;p&gt;Thank you Roman !!!&lt;/p&gt;</comment>
                            <comment id="1234503" author="ramon.fernandez" created="Wed, 13 Apr 2016 13:19:24 +0000"  >&lt;p&gt;We were not able to reproduce the behavior described in this ticket, but as pointed out above it could be related to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-21275&quot; title=&quot;Document not found due to WT commit visibility issue&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-21275&quot;&gt;&lt;del&gt;SERVER-21275&lt;/del&gt;&lt;/a&gt;. I&apos;d recommend upgrading to 3.0.11 and, if the problem persists, open a new SERVER ticket.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1234439" author="saisrinivase@gmail.com" created="Wed, 13 Apr 2016 12:55:57 +0000"  >&lt;p&gt;Hi Ramon, &lt;/p&gt;

&lt;p&gt;Thank you , i have already started the re-sync to make its a secondary.&lt;/p&gt;

&lt;p&gt;I am glad to know about the FATAL ASSERTION with duplicate key error , because that is the one i am not able to find out .&lt;/p&gt;

&lt;p&gt;Could you help me on this , appreciate your help !!!!&lt;/p&gt;</comment>
                            <comment id="1234436" author="ramon.fernandez" created="Wed, 13 Apr 2016 12:53:18 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=saisrinivase%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;saisrinivase@gmail.com&quot;&gt;saisrinivase@gmail.com&lt;/a&gt;, it seems this node has become too stale to be able to catch up, and you&apos;ll need to &lt;a href=&quot;https://docs.mongodb.org/manual/tutorial/resync-replica-set-member/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;resync it&lt;/a&gt;. If the problem persists please open a new ticket so we can investigate it separately.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1234324" author="saisrinivase@gmail.com" created="Wed, 13 Apr 2016 09:05:49 +0000"  >&lt;p&gt;Again today it&apos;s re-sync failed with the below error , &lt;/p&gt;

&lt;p&gt;2016-04-12T20:45:59.697-0400 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: xxxxxxxxxx:27015                                                                           &lt;br/&gt;
2016-04-12T20:45:59.778-0400 W REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; we are too stale to use xxxxxxx:27015 as a sync source                                                   &lt;br/&gt;
2016-04-12T20:45:59.778-0400 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: xxxxxxxxxxx:27015                                                                           &lt;br/&gt;
2016-04-12T20:45:59.833-0400 W REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; we are too stale to use xxxxxxxxxx:27015 as a sync source                                                   &lt;br/&gt;
2016-04-12T20:45:59.833-0400 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from:xxxxxxxxxxxxxxxx:27015                                                                           &lt;br/&gt;
2016-04-12T20:45:59.886-0400 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; could not find member to sync from                                                                                &lt;br/&gt;
2016-04-12T20:45:59.886-0400 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; replSet error RS102 too stale  to catch up                                                                            &lt;br/&gt;
2016-04-12T20:45:59.886-0400 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; replSet our last optime : Apr  11 09:00:36 570b9ff4:1dc                                                               &lt;br/&gt;
2016-04-12T20:45:59.886-0400 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; replSet oldest available is Apr 11 15:41:34 570bfdee:139                                                             &lt;br/&gt;
2016-04-12T20:45:59.886-0400 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember&lt;/a&gt;             &lt;/p&gt;

&lt;p&gt;016-04-12T20:46:20.811-0400 I QUERY    &lt;span class=&quot;error&quot;&gt;&amp;#91;conn15428&amp;#93;&lt;/span&gt; assertion 13436 not master or second&lt;br/&gt;
ary; cannot currently read from this replSet member ns:config.settings query:{}         &lt;br/&gt;
2016-04-12T20:46:24.096-0400 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;conn15539&amp;#93;&lt;/span&gt; end connection 10.160.137.7:53778 (16 connections now open)&lt;/p&gt;

&lt;p&gt;2016-04-12T20:47:15.724-0400 I QUERY    &lt;span class=&quot;error&quot;&gt;&amp;#91;conn15428&amp;#93;&lt;/span&gt; assertion 13436 not master or second&lt;br/&gt;
ary; cannot currently read from this replSet member ns:config.settings query:{}         &lt;br/&gt;
2016-04-12T20:47:24.947-0400 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;conn15547&amp;#93;&lt;/span&gt; end connection 10.160.137.7:53794 (17 connections now open)&lt;/p&gt;</comment>
                            <comment id="1233108" author="shankar.k" created="Tue, 12 Apr 2016 12:29:41 +0000"  >&lt;p&gt;I think this issue is related to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-21275&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-21275&lt;/a&gt; &lt;br/&gt;
It might be solved If you upgrade your  mongodb to 3.0.8 or greater&lt;/p&gt;</comment>
                            <comment id="1233032" author="saisrinivase@gmail.com" created="Tue, 12 Apr 2016 10:56:00 +0000"  >&lt;p&gt;2016-03-23T20:15:51.366-0400 I -        &lt;span class=&quot;error&quot;&gt;&amp;#91;repl writer worker 2&amp;#93;&lt;/span&gt; Fatal Assertion 16361&lt;/p&gt;

&lt;p&gt;Error message as above , please help me on this ..Thanks in advance !!!!&lt;/p&gt;</comment>
                            <comment id="1232005" author="saisrinivase@gmail.com" created="Mon, 11 Apr 2016 15:37:02 +0000"  >&lt;p&gt;Do we have any solution for this in 3.0.7 version .&lt;/p&gt;

&lt;p&gt;I would like thanks and appreciate if we have any solution for this error.&lt;/p&gt;

&lt;p&gt;Since i have started doing re-sync twice but at last it is failing with the same error as posted by Shankar.&lt;/p&gt;</comment>
                            <comment id="1077753" author="jongkim@ea.com" created="Tue, 3 Nov 2015 07:03:36 +0000"  >&lt;p&gt;We&apos;re experiencing same issues what original author mentioned. I think I can answer to the question..&lt;/p&gt;

&lt;p&gt;1. No&lt;br/&gt;
2. Local RAID Volume. No recent failures recently.&lt;br/&gt;
3. Yes, not just one or two node. We have experienced 5-6 times a week.&lt;/p&gt;

&lt;p&gt;We installed 3.0.7 on every node from the beginning, and migrated data from the nodes running 2.4.13.&lt;br/&gt;
We&apos;re using WiredTiger as storage engine, using snappy compression, no journaling.&lt;/p&gt;</comment>
                            <comment id="1050494" author="ramon.fernandez" created="Sat, 3 Oct 2015 12:51:37 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=shankar.k&quot; class=&quot;user-hover&quot; rel=&quot;shankar.k&quot;&gt;shankar.k&lt;/a&gt;, we haven&apos;t heard back from you for a while so we&apos;re closing this ticket. If this is still an issue for you please provide the additional information requested by Sam above.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1019447" author="samk" created="Tue, 1 Sep 2015 15:31:00 +0000"  >&lt;p&gt;Sorry for the delay in getting back to you. I&apos;ve discussed this case with a few of my colleagues and we have three possible explanations for the error that you saw: &lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;The data on the secondary was modified during a maintenance operation where the instance was running in &quot;standalone&quot; mode and not a member of the set.&lt;/li&gt;
	&lt;li&gt;Data corruption on the secondary member of the set, as a result of storage (block device) system failure, networking errors, or other modification.&lt;/li&gt;
	&lt;li&gt;A logical error in the replication system.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;In most cases the clock skew should &lt;b&gt;not&lt;/b&gt; be a problem, although running a time synchronization service (e.g. &lt;tt&gt;ntpd&lt;/tt&gt;) may be a good idea as a matter of general practice. There are some cases where the combination of clock skew and unreliable networks can produce a replica set with multiple primaries for a short period of time. This is confusing for the client driver, but should lead to a rollback rather than the kind of error that you see. &lt;/p&gt;

&lt;p&gt;In an attempt to better understand what&apos;s happening here, could you answer the following questions: &lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;Did you preform any maintenance on this member of the replica set, at any point in its history? Was there a period where this member was running but not a member of the replica set (i.e. ruining without the &lt;tt&gt;replSet&lt;/tt&gt; option,)  where a client might have been able to connect and modify the data (even unintentionally?)&lt;/li&gt;
	&lt;li&gt;What kind of storage system are the data file stored in? Is it local or attached via a networking interface? Are you aware of recent drive failures? Have you had previous issues with storage consistency?&lt;/li&gt;
	&lt;li&gt;Has this issue continued to affect your deployment? Have you had other nodes affected by this issue? Has the same node continued to be affected by this issue? Do you have an idea of which document or documents are triggering this error? We&apos;d like to figure out a way to reproduce this issue. Do you still have a copy of the data where you see this error?&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;Sorry again for the delay, and thanks for your help. &lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
sam&lt;/p&gt;</comment>
                            <comment id="1003512" author="ramon.fernandez" created="Wed, 12 Aug 2015 18:42:06 +0000"  >&lt;p&gt;Thanks for uploading the logs &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=shankar.k&quot; class=&quot;user-hover&quot; rel=&quot;shankar.k&quot;&gt;shankar.k&lt;/a&gt;; this is what I see on the primary:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:48:01.310+0000 I COMMAND  [conn6452433] command talk-vsession.$cmd command: delete { delete: &quot;visitorSessions&quot;, &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;writeConcern: { w: &quot;majority&quot;, wtimeout: 5000 }, ordered: true, deletes: [ { q: { _id: { $in: [ ObjectId(&apos;55c05d77d896ff11c05b&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;e9a6&apos;), ObjectId(&apos;55c05da8d896ff11c05bf1cf&apos;), ObjectId(&apos;55c05d14d896ff11c05bd8d6&apos;), ObjectId(&apos;55c05c57d896ff11c05bb85e&apos;), ObjectId(&apos;55c05dbdd896ff11c05bf53b&apos;), ObjectId(&apos;55c05b06d896ff11c05b7fee&apos;), ObjectId(&apos;55c057ecd896ff11c05aff26&apos;), ObjectId(&apos;55c05893d896ff11c05b1a26&apos;), ObjectId(&apos;55c05db8d896ff11c05bf463&apos;), ObjectId(&apos;55c05d46d896ff11c05be13a&apos;), ObjectId(&apos;55c05986d896ff11c05b4141&apos;), ObjectId(&apos;55c05d87d896ff11c05bec3b&apos;), ObjectId(&apos;55c05dbad896ff11c05bf4b6&apos;), ObjectId(&apos;55c05d78d896ff11c05be9c8&apos;), ObjectId(&apos;55c05d94d896ff11c05bee83&apos;), ObjectId(&apos;55c05d26d896ff11c05bdbdf&apos;), ObjectId(&apos;55c05d42d896ff11c05be083&apos;), ObjectId(&apos;55c05c21d896ff11c05baee7&apos;), ObjectId(&apos;55c05d9fd896ff11c05bf07d&apos;), ObjectId(&apos;55c05d87d896ff11c05bec56&apos;), ObjectId(&apos;55c05d95d896ff11c05bee9f&apos;), ObjectId(&apos;55c05dbcd896ff11c05bf527&apos;), ObjectId(&apos;55c05dbbd896ff11c05bf4f1&apos;), ObjectId(&apos;55c05d47d896ff11c05be141&apos;), ObjectId(&apos;55c04478d896ff11c0582d7d&apos;), ObjectId(&apos;55c05d17d896ff11c05bd966&apos;), ObjectId(&apos;55c05d5fd896ff11c05be563&apos;), ObjectId(&apos;55c05d81d896ff11c05beb4e&apos;), ObjectId(&apos;55c05dbdd896ff11c05bf548&apos;), ObjectId(&apos;55c05973d896ff11c05b3e35&apos;), ObjectId(&apos;55c05d87d896ff11c05bec4e&apos;), ObjectId(&apos;55c05db9d896ff11c05bf49d&apos;), ObjectId(&apos;55c0594bd896ff11c05b37c7&apos;), ObjectId(&apos;55c04f8fd896ff11c059b8f9&apos;), ObjectId(&apos;55c05d00d896ff11c05bd58f&apos;), ObjectId(&apos;55c049efd896ff11c058eb16&apos;), ObjectId(&apos;55c05c95d896ff11c05bc305&apos;), ObjectId(&apos;55c05d78d896ff11c05be9cd&apos;), ObjectId(&apos;55c04fb4d896ff11c059be68&apos;), ObjectId(&apos;55c05d03d896ff11c05bd62a&apos;), ObjectId(&apos;55c05d76d896ff11c05be983&apos;), ObjectId(&apos;55c05d79d896ff11c05be9e3&apos;), ObjectId(&apos;55c048afd896ff11c058be66&apos;), ObjectId(&apos;55c05da5d896ff11c05bf15e&apos;), ObjectId(&apos;55c05bddd896ff11c05ba338&apos;), ObjectId(&apos;55c05c86d896ff11c05bc05c&apos;), ObjectId(&apos;55c05b8ad896ff11c05b9544&apos;), ObjectId(&apos;55c05d88d896ff11c05bec60&apos;), ObjectId(&apos;55c05da4d896ff11c05bf149&apos;), ObjectId(&apos;55c05a29d896ff11c05b5bc8&apos;) ] } }, limit: 0 } ] } keyUpdates:0 writeConflicts:0 numYields:0 reslen:185 locks:{ Global: { acquireCount: { r: 51, w: 51 } }, Database: { acquireCount: { w: 51 } }, Collection: { acquireCount: { w: 1 } }, oplog: { acquireCount: { w: 50 } } } 5019ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:48:01.310+0000 I NETWORK  [conn6452433] end connection 10.96.4.137:50583 (3041 connections now open)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:48:01.602+0000 I COMMAND  [conn3261683] command talk-asession.$cmd command: update { update: &quot;agentSessions&quot;, writeConcern: { w: &quot;majority&quot;, wtimeout: 15000 }, ordered: true, updates: [ { q: { _id: ObjectId(&apos;55c053fdac9121063ec28f86&apos;), eo: { $gt: new Date(1438670775312) } }, u: { $set: { eo: new Date(1438672575312), uo: new Date(1438670775312) } }, multi: false, upsert: false } ] } keyUpdates:0 writeConflicts:0 numYields:0 reslen:200 locks:{ Global: { acquireCount: { r: 2, w: 2 } }, Database: { acquireCount: { w: 2 } }, Collection: { acquireCount: { w: 1 } }, oplog: { acquireCount: { w: 1 } } } 15001ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:48:01.729+0000 W NETWORK  [ReplExecNetThread-402] Failed to connect to 10.97.176.65:27017, reason: errno:111 Connection refused&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:48:01.729+0000 I REPL     [ReplicationExecutor] Error in heartbeat request to mongodb-alpha-1:27017; Location18915 Failed attempt to connect to mongodb-alpha-1:27017; couldn&apos;t connect to server mongodb-alpha-1:27017 (10.97.176.65), connection attempt failed&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Right after the delete command the secondary &lt;tt&gt;mongodb-alpha-1&lt;/tt&gt; becomes unreachable; in the &lt;tt&gt;mongodb-alpha-1&lt;/tt&gt; logs I see:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:37:49.530+0000 E REPL     [repl writer worker 9] writer worker caught exception:  :: caused by :: 11000 E11000 duplicate key error collection: talk-vsession.visitorSessions index: _id_ dup key: { : ObjectId(&apos;55c05dbdd896ff11c05bf548&apos;) } on: { ts: Timestamp 1438670269000|632, h: -3999426385275226819, v: 2, op: &quot;u&quot;, ns: &quot;talk-vsession.visitorSessions&quot;, o2: { _id: ObjectId(&apos;55c05dbdd896ff11c05bf548&apos;) }, o: { $set: { sm: &quot;e56d8d2df5c4e6761a36b7e88efe0681853a6f54&quot;, uo: new Date(1438670269476) } } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:37:49.530+0000 I -        [repl writer worker 9] Fatal Assertion 16360&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T06:37:49.530+0000 I -        [repl writer worker 9] &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;***aborting after fassert() failure&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-08-04T07:01:24.391+0000 I CONTROL  ***** SERVER RESTARTED *****&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;Both commands mention the offending objectid, &lt;tt&gt;55c05dbdd896ff11c05bf548&lt;/tt&gt;, so I&apos;m trying to understand how the &lt;tt&gt;delete&lt;/tt&gt; command may trigger this assertion on the secondary node.&lt;/p&gt;

&lt;p&gt;I&apos;m very suspicious of the clock difference between these two nodes, more than 10 minutes. I don&apos;t know if this is the source of the problem yet or just a coincidence.&lt;/p&gt;

&lt;p&gt;We&apos;ll investigate this further and let you know if we need additional information. I see that the &lt;tt&gt;mongodb-alpha-1&lt;/tt&gt; successfully transitioned to secondary after a restart; I also see a number of restarts on this system, are you still affected by this issue or are the restarts expected?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="998263" author="shankar.k" created="Thu, 6 Aug 2015 14:00:01 +0000"  >&lt;p&gt;Hello Ramon,&lt;/p&gt;

&lt;p&gt;Thank you for creating the bucket. I have upload the logs&lt;br/&gt;
mongod-alpha-3.log.2015-08-05T00-00-01.tar.gz =&amp;gt; Primary&lt;br/&gt;
mongod-alpha-1.log.2015-08-05T00-00-01.tar.gz =&amp;gt; Secondary that crashed&lt;/p&gt;

&lt;p&gt;For info about load, maybe you checkout our MMS account, organization name  is &lt;tt&gt;loveclients&lt;/tt&gt;&lt;br/&gt;
I&apos;m not sure about the type of operations that it was running when it crashed.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Shankar&lt;/p&gt;</comment>
                            <comment id="998252" author="ramon.fernandez" created="Thu, 6 Aug 2015 13:49:54 +0000"  >&lt;p&gt;Here it is: &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/e108a7f2-b1d4-492c-96dd-841c1ff25316.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/e108a7f2-b1d4-492c-96dd-841c1ff25316.html&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="998246" author="shankar.k" created="Thu, 6 Aug 2015 13:47:33 +0000"  >&lt;p&gt;Hello Ramon,&lt;/p&gt;

&lt;p&gt;Could you create a MongoDB Support File Upload bucket, please ?&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Shankar&lt;/p&gt;</comment>
                            <comment id="998166" author="ramon.fernandez" created="Thu, 6 Aug 2015 12:20:14 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=shankar.k&quot; class=&quot;user-hover&quot; rel=&quot;shankar.k&quot;&gt;shankar.k&lt;/a&gt;, could you please upload full logs for the primary node as well as the affected secondary? Are there other details you can provide as of to how this assertion was triggered (load, type of operations, etc.)?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="223081">SERVER-19633</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>21.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 6 Aug 2015 12:18:17 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 40 weeks, 4 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>milkie@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 40 weeks, 4 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>jongkim@ea.com</customfieldvalue>
            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>
            <customfieldvalue>sachinrohdia@gmail.com</customfieldvalue>
            <customfieldvalue>saisrinivase@gmail.com</customfieldvalue>
            <customfieldvalue>sam.kleinman</customfieldvalue>
            <customfieldvalue>shankar.k</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrkycf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hscd13:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrjdin:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>