<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:26:31 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-11698] Fatal Assertion during replication</title>
                <link>https://jira.mongodb.org/browse/SERVER-11698</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;One of the replicas got fatal assertion 16360 during replication:&lt;/p&gt;

&lt;p&gt;Thu Nov 14 07:44:15.896 &lt;span class=&quot;error&quot;&gt;&amp;#91;repl writer worker 1&amp;#93;&lt;/span&gt;  production.messages Assertion failure type == cbindata src/mongo/db/key.cpp 585&lt;br/&gt;
0xde05e1 0xda15bd 0xa2a7fc 0x7fa6ed 0x7fd056 0x7fdefd 0x7fc8c3 0x7fc9ab 0x7fc9ab 0x7fc9ab 0x7fc9ab 0x8009ea 0x9d75bb 0x9e03fa 0xac587d 0xac6b7f 0xa9168a 0xa93757 0xa6d7a0 0xc29713&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xde05e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo12verifyFailedEPKcS1_j+0xfd) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xda15bd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo5KeyV18dataSizeEv+0x12c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa2a7fc&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo12BucketBasicsINS_12BtreeData_V1EE16_packReadyForModERKNS_8OrderingERi+0xdd) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fa6ed&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11BtreeBucketINS_12BtreeData_V1EE5splitENS_7DiskLocEiS3_RKNS_5KeyV1ERKNS_8OrderingES3_S3_RNS_12IndexDetailsE+0x5a6) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fd056&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo11BtreeBucketINS_12BtreeData_V1EE10insertHereENS_7DiskLocEiS3_RKNS_5KeyV1ERKNS_8OrderingES3_S3_RNS_12IndexDetailsE+0xabd) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fdefd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo11BtreeBucketINS_12BtreeData_V1EE7_insertENS_7DiskLocES3_RKNS_5KeyV1ERKNS_8OrderingEbS3_S3_RNS_12IndexDetailsE+0x363) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fc8c3&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo11BtreeBucketINS_12BtreeData_V1EE7_insertENS_7DiskLocES3_RKNS_5KeyV1ERKNS_8OrderingEbS3_S3_RNS_12IndexDetailsE+0x44b) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fc9ab&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo11BtreeBucketINS_12BtreeData_V1EE7_insertENS_7DiskLocES3_RKNS_5KeyV1ERKNS_8OrderingEbS3_S3_RNS_12IndexDetailsE+0x44b) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fc9ab&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo11BtreeBucketINS_12BtreeData_V1EE7_insertENS_7DiskLocES3_RKNS_5KeyV1ERKNS_8OrderingEbS3_S3_RNS_12IndexDetailsE+0x44b) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fc9ab&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo11BtreeBucketINS_12BtreeData_V1EE7_insertENS_7DiskLocES3_RKNS_5KeyV1ERKNS_8OrderingEbS3_S3_RNS_12IndexDetailsE+0x44b) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fc9ab&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo11BtreeBucketINS_12BtreeData_V1EE9bt_insertENS_7DiskLocES3_RKNS_7BSONObjERKNS_8OrderingEbRNS_12IndexDetailsEb+0x11a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8009ea&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo18IndexInterfaceImplINS_12BtreeData_V1EE9bt_insertENS_7DiskLocES3_RKNS_7BSONObjERKNS_8OrderingEbRNS_12IndexDetailsEb+0x8b) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9d75bb&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo24indexRecordUsingTwoStepsEPKcPNS_16NamespaceDetailsENS_7BSONObjENS_7DiskLocEb+0xbca) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9e03fa&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11DataFileMgr6insertEPKcPKvibbbPb+0x123d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xac587d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11DataFileMgr16insertWithObjModEPKcRNS_7BSONObjEbb+0x4f) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xac6b7f&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEPNS_11RemoveSaverEbRKNS_24QueryPlanSelectionPolicyEb+0x2eda) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa9168a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo27updateObjectsForReplicationEPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEbRKNS_24QueryPlanSelectionPolicyE+0xb7) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa93757&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo21applyOperation_inlockERKNS_7BSONObjEbb+0x600) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa6d7a0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo7replset8SyncTail9syncApplyERKNS_7BSONObjEb+0x713) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xc29713&amp;#93;&lt;/span&gt;&lt;br/&gt;
Thu Nov 14 07:44:15.912 &lt;span class=&quot;error&quot;&gt;&amp;#91;repl writer worker 1&amp;#93;&lt;/span&gt; ERROR: writer worker caught exception: assertion src/mongo/db/key.cpp:585 on: { ts: Timestamp 1384415055000|7, h: 5648881096906150481, v: 2, op: &quot;i&quot;, ns: &quot;production.messages&quot;, o: &lt;/p&gt;
{ _id: ObjectId(&apos;52847f4f3032ca9060ab9c95&apos;), ... }
&lt;p&gt; }&lt;br/&gt;
Thu Nov 14 07:44:15.912 &lt;span class=&quot;error&quot;&gt;&amp;#91;repl writer worker 1&amp;#93;&lt;/span&gt;   Fatal Assertion 16360&lt;br/&gt;
0xde05e1 0xda03d3 0xc28f3c 0xdadf21 0xe28e69 0x7f793cbb5e9a 0x7f793bec83fd&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xde05e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo13fassertFailedEi+0xa3) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xda03d3&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo7replset14multiSyncApplyERKSt6vectorINS_7BSONObjESaIS2_EEPNS0_8SyncTailE+0x12c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xc28f3c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo10threadpool6Worker4loopEv+0x281) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xdadf21&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xe28e69&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/x86_64-linux-gnu/libpthread.so.0(+0x7e9a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f793cbb5e9a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f793bec83fd&amp;#93;&lt;/span&gt;&lt;br/&gt;
Thu Nov 14 07:44:15.913 &lt;span class=&quot;error&quot;&gt;&amp;#91;repl writer worker 1&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;***aborting after fassert() failure&lt;/p&gt;


&lt;p&gt;Thu Nov 14 07:44:15.913 Got signal: 6 (Aborted).&lt;/p&gt;

&lt;p&gt;Thu Nov 14 07:44:15.915 Backtrace:&lt;br/&gt;
0xde05e1 0x6d0559 0x7f793be0a4a0 0x7f793be0a425 0x7f793be0db8b 0xda040e 0xc28f3c 0xdadf21 0xe28e69 0x7f793cbb5e9a 0x7f793bec83fd&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xde05e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo10abruptQuitEi+0x399) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6d0559&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/x86_64-linux-gnu/libc.so.6(+0x364a0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f793be0a4a0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/x86_64-linux-gnu/libc.so.6(gsignal+0x35) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f793be0a425&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/x86_64-linux-gnu/libc.so.6(abort+0x17b) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f793be0db8b&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo13fassertFailedEi+0xde) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xda040e&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo7replset14multiSyncApplyERKSt6vectorINS_7BSONObjESaIS2_EEPNS0_8SyncTailE+0x12c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xc28f3c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo10threadpool6Worker4loopEv+0x281) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xdadf21&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xe28e69&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/x86_64-linux-gnu/libpthread.so.0(+0x7e9a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f793cbb5e9a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f793bec83fd&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;After that the mongod no longer starts, just crashes to the same thing every time.&lt;/p&gt;

&lt;p&gt;The document contained an (indexed) array field where one entry was &quot;sch&#246;ne&quot;, so it included a non-ascii character, hard to say if it&apos;s relevant. The other replica handled it just fine.&lt;/p&gt;

&lt;p&gt;We&apos;ve seen these replication errors every few months, previously with 2.2.x and now with 2.4.x. It takes a week to resync a replica, so is it safe to just insert that document (or empty one with the same ts and h) manually to the local&apos;s db.oplog.rs and it would then continue replicating normally?&lt;/p&gt;</description>
                <environment>Linux, ubuntu 12.04 64bit</environment>
        <key id="98418">SERVER-11698</key>
            <summary>Fatal Assertion during replication</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="-1">Unassigned</assignee>
                                    <reporter username="devastor">Tuomas Silen</reporter>
                        <labels>
                    </labels>
                <created>Thu, 14 Nov 2013 10:21:15 +0000</created>
                <updated>Wed, 10 Dec 2014 23:19:00 +0000</updated>
                            <resolved>Wed, 19 Mar 2014 04:22:48 +0000</resolved>
                                    <version>2.4.8</version>
                                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="518318" author="devastor" created="Tue, 18 Mar 2014 13:56:54 +0000"  >&lt;p&gt;Hi Stephen,&lt;/p&gt;

&lt;p&gt;Yes, sorry for not updating the ticket. We ended up replacing the disks after the issue kept repeating and after that we haven&apos;t seen it again. So it would seem like it did indeed have something to do with the disks although smart, swraid, fsck, etc. didn&apos;t find anything and there was nothing in logs/dmesg.&lt;/p&gt;

&lt;p&gt;Feel free to close the ticket.&lt;/p&gt;</comment>
                            <comment id="518125" author="stennie" created="Tue, 18 Mar 2014 04:33:35 +0000"  >&lt;p&gt;Hi Tuomas,&lt;/p&gt;

&lt;p&gt;Apologies for the delay in follow-up .. are you still seeing this issue, or were you able to resolve?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Stephen&lt;/p&gt;</comment>
                            <comment id="458103" author="devastor" created="Mon, 18 Nov 2013 21:43:34 +0000"  >&lt;p&gt;There are no disk errors in syslog/dmesg and smart doesn&apos;t show any errors. The disks are about 5 month old SSD disks.&lt;/p&gt;</comment>
                            <comment id="457919" author="eliot" created="Mon, 18 Nov 2013 17:57:27 +0000"  >&lt;p&gt;Can you check the system logs for disk errors?&lt;/p&gt;</comment>
                            <comment id="456778" author="devastor" created="Fri, 15 Nov 2013 12:10:18 +0000"  >&lt;p&gt;We entered that entry to the oplog and it then continued to successfully replicate again, but after a day the same mongod crashed again to another document (also that one contained &#246; and &#252; characters). So I guess the index is just somehow corrupted there. &lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 18 Nov 2013 17:57:27 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        9 years, 48 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            9 years, 48 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>eliot</customfieldvalue>
            <customfieldvalue>stephen.steneker@mongodb.com</customfieldvalue>
            <customfieldvalue>devastor</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrm7jj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrwlfj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>104206</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;unknown&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hsp51z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>