<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 09:06:05 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[KAFKA-313] Sink Connector Configuration on sharded collection</title>
                <link>https://jira.mongodb.org/browse/KAFKA-313</link>
                <project id="16285" key="KAFKA">Kafka Connector</project>
                    <description>&lt;p&gt;We are trying to use Mongo Sink Connector for Kafka to sink the messages from a Kafka topic. Currently we are using default configuration, i.e., the default write model strategy. This configuration seems to work for a normal collection but for sharded collection we are encountering an error.&lt;/p&gt;

&lt;p&gt;Write errors: &lt;span class=&quot;error&quot;&gt;&amp;#91;BulkWriteError\{index=0, code=61, message=&amp;#39;Failed to target upsert by query :: could not extract exact shard key&amp;#39;, details={}}&amp;#93;&lt;/span&gt;.&lt;/p&gt;

&lt;p&gt;Was not able to find enough documentation of what needs to be provided for a sharded collection. The intent here is to work in a similar manner as the default ReplaceOneDefaultStrategy.&lt;/p&gt;

&lt;p&gt;Please let us know the writemodel strategy that would be applicable for sharded collection.&lt;/p&gt;</description>
                <environment></environment>
        <key id="2040520">KAFKA-313</key>
            <summary>Sink Connector Configuration on sharded collection</summary>
                <type id="6" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14720&amp;avatarType=issuetype">Question</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13203">Gone away</resolution>
                                        <assignee username="robert.walters@mongodb.com">Robert Walters</assignee>
                                    <reporter username="rajesh.vinayagam@mongodb.com">Rajesh Vinayagam</reporter>
                        <labels>
                    </labels>
                <created>Thu, 5 May 2022 16:53:05 +0000</created>
                <updated>Fri, 27 Oct 2023 19:45:05 +0000</updated>
                            <resolved>Tue, 11 Apr 2023 12:01:03 +0000</resolved>
                                    <version>1.7.0</version>
                                                    <component>Configuration</component>
                    <component>Documentation</component>
                    <component>Sink</component>
                                        <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="5337733" author="dbeng-pm-bot" created="Tue, 11 Apr 2023 12:01:05 +0000"  >&lt;p&gt;There hasn&apos;t been any recent activity on this ticket, so we&apos;re resolving it. Thanks for reaching out! Please feel free to comment on this if you&apos;re able to provide more information.&lt;/p&gt;</comment>
                            <comment id="5322769" author="dbeng-pm-bot" created="Tue, 4 Apr 2023 12:00:47 +0000"  >&lt;p&gt;Hey &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=rajesh.vinayagam%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;rajesh.vinayagam@mongodb.com&quot;&gt;rajesh.vinayagam@mongodb.com&lt;/a&gt;, We need additional details to investigate the problem. If this is still an issue for you, please provide the requested information.&lt;/p&gt;</comment>
                            <comment id="5302721" author="jeff.yemin" created="Mon, 27 Mar 2023 14:35:46 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=rajesh.vinayagam%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;rajesh.vinayagam@mongodb.com&quot;&gt;rajesh.vinayagam@mongodb.com&lt;/a&gt; should we consider this resolved then?&lt;/p&gt;</comment>
                            <comment id="4538824" author="JIRAUSER1259084" created="Tue, 10 May 2022 16:45:37 +0000"  >&lt;p&gt;Hi Ross.&lt;/p&gt;

&lt;p&gt;&lt;del&gt;I tried doing that as well document.id.strategy.partial.value.projection.list, with that the connector was erroring out, so removed that from the list.&lt;/del&gt;&lt;/p&gt;

&lt;p&gt;Removed the connector and reinitialized with the above settings and it was able to replace the document&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;br/&gt;
&quot;document.id.strategy.partial.value.projection.list&quot;: &quot;xxx,xxx,_id&quot;,&lt;br/&gt;
&#160;&lt;br/&gt;
Rajesh&lt;/p&gt;</comment>
                            <comment id="4537402" author="ross@10gen.com" created="Tue, 10 May 2022 08:16:50 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=rajesh.vinayagam%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;rajesh.vinayagam@mongodb.com&quot;&gt;rajesh.vinayagam@mongodb.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;The issue is the &lt;tt&gt;ReplaceOneBusinessKeyStrategy&lt;/tt&gt; removes the &lt;tt&gt;_id&lt;/tt&gt; field if present due to the value of the identifier coming from the supplied projection list.  &lt;/p&gt;

&lt;p&gt;Can you include the _id in the &lt;tt&gt;document.id.strategy.partial.value.projection.list&lt;/tt&gt; ? If  not I think for sharded clusters a custom write strategy may be required to handle providing the shardkey and existing keys. If it works then an example should be added to the documentation to help users in the future.&lt;/p&gt;

&lt;p&gt;Ross&lt;/p&gt;</comment>
                            <comment id="4536389" author="JIRAUSER1259084" created="Mon, 9 May 2022 19:25:42 +0000"  >&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;A quick update on the above issue, I further tried with different configuration and&#160;I was able to get it work with the below configuration &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160; &#160;&lt;/p&gt;

&lt;p&gt;&#160; &#160; &lt;b&gt;&lt;em&gt;&quot;writemodel.strategy&quot;:&quot;com.mongodb.kafka.connect.sink.writemodel.strategy.ReplaceOneBusinessKeyStrategy&quot;,&lt;/em&gt;&lt;/b&gt;&lt;br/&gt;
&#160; &#160; &lt;b&gt;&lt;em&gt;&quot;document.id.strategy&quot;: &quot;com.mongodb.kafka.connect.sink.processor.id.strategy.PartialValueStrategy&quot;,&lt;/em&gt;&lt;/b&gt;&lt;br/&gt;
&#160; &#160; &lt;b&gt;&lt;em&gt;&quot;document.id.strategy.overwrite.existing&quot;: &quot;true&quot;,&lt;/em&gt;&lt;/b&gt;&lt;br/&gt;
&#160; &#160; &lt;b&gt;&lt;em&gt;&quot;document.id.strategy.partial.value.projection.type&quot;: &quot;allowlist&quot;,&lt;/em&gt;&lt;/b&gt;&lt;br/&gt;
&#160; &#160; &lt;b&gt;&lt;em&gt;&quot;document.id.strategy.partial.value.projection.list&quot;: &quot;xxx,xxx&quot;&lt;/em&gt;&lt;/b&gt;&#160;&lt;/p&gt;

&lt;p&gt;The issue I had with above configuration was the _id was not retained from the source document instead it was generating a new _id&lt;/p&gt;

&lt;p&gt;But if the shard key matches i.e, the values mentioned in the projetion list it was able to update the existing document.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Is there a way to retain the _id from the source document instead of generating a new _id when inserting or updating with the above configuration.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i0csiy:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                </customfields>
    </item>
</channel>
</rss>