<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 09:05:31 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[KAFKA-78] Publish error messages to a topic</title>
                <link>https://jira.mongodb.org/browse/KAFKA-78</link>
                <project id="16285" key="KAFKA">Kafka Connector</project>
                    <description>&lt;p&gt;In case of failure to process events (conversion failing, retryable errors, etc) on the connector side, we should have the ability to publish a message to an error topic with sufficient details to identify a potential series of missed events.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1065917">KAFKA-78</key>
            <summary>Publish error messages to a topic</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="ross@mongodb.com">Ross Lawley</assignee>
                                    <reporter username="davenson.lombard@mongodb.com">Davenson Lombard</reporter>
                        <labels>
                    </labels>
                <created>Tue, 24 Dec 2019 19:24:09 +0000</created>
                <updated>Sat, 28 Oct 2023 10:46:24 +0000</updated>
                            <resolved>Fri, 11 Sep 2020 12:53:09 +0000</resolved>
                                                    <fixVersion>1.3.0</fixVersion>
                                    <component>Source</component>
                                        <votes>2</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="3387032" author="xgen-internal-githook" created="Fri, 11 Sep 2020 12:46:35 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Ross Lawley&apos;, &apos;email&apos;: &apos;ross.lawley@gmail.com&apos;, &apos;username&apos;: &apos;rozza&apos;}
&lt;p&gt;Message: Added dead letter queue support for the source connector&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/KAFKA-78&quot; title=&quot;Publish error messages to a topic&quot; class=&quot;issue-link&quot; data-issue-key=&quot;KAFKA-78&quot;&gt;&lt;del&gt;KAFKA-78&lt;/del&gt;&lt;/a&gt;&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo-kafka/commit/d68d7da45c98148bed2fef00ef2cf5ba64031d3a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo-kafka/commit/d68d7da45c98148bed2fef00ef2cf5ba64031d3a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3378858" author="ross@10gen.com" created="Tue, 8 Sep 2020 15:15:06 +0000"  >&lt;p&gt;PR: &lt;a href=&quot;https://github.com/mongodb/mongo-kafka/pull/36&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo-kafka/pull/36&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3049191" author="yaramati@adobe.com" created="Tue, 21 Apr 2020 15:14:04 +0000"  >&lt;p&gt;I think other way. deadletter. sink has deadletterqueue option not source.&#160;&lt;a href=&quot;https://kafka.apache.org/documentation/#errors.deadletterqueue.topic.name&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://kafka.apache.org/documentation/#errors.deadletterqueue.topic.name&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2731390" author="davenson.lombard" created="Wed, 15 Jan 2020 14:40:04 +0000"  >&lt;p&gt;Nevermind about my previous comment. The dead letter queue is available for source connectors, not sink.&lt;/p&gt;</comment>
                            <comment id="2715073" author="davenson.lombard" created="Thu, 9 Jan 2020 19:55:12 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=ross.lawley&quot; class=&quot;user-hover&quot; rel=&quot;ross.lawley&quot;&gt;ross.lawley&lt;/a&gt;, I completely agree with you.&lt;/p&gt;

&lt;p&gt;I don&apos;t have much experience with Kafka itself and most of the knowledge I have is coming from the Confluent blog posts and documentation. In looking for a potential solution to this customer request,  I found that since Kafka 2.0, Kafka Connect includes some error handling mechanism. One can take advantage of it to write the messages that can&apos;t be processed to a &quot;dead letter queue&quot;. &lt;/p&gt;

&lt;p&gt;More details here: &lt;a href=&quot;https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues/&lt;/a&gt; &lt;/p&gt;

&lt;p&gt;If I read this correctly, by setting &lt;tt&gt;errors.tolerance&lt;/tt&gt; and &lt;tt&gt;errors.deadletterqueue.topic.name&lt;/tt&gt; in the connector config, one can have the &quot;Failed messages&quot; sent to the dead letter queue.&lt;/p&gt;

&lt;p&gt;If this works as described, I believe it will be sufficient to have errors and message that we failed to process be sent directly to a queue with a name defined by the customer.&lt;/p&gt;
</comment>
                            <comment id="2705197" author="ross@10gen.com" created="Mon, 6 Jan 2020 15:22:12 +0000"  >&lt;p&gt;I&apos;m not sure exactly what should be published and / or if this a pattern that other Kafka connectors follow. It would be good to follow any existing conventions rather than introducing any new paradigms.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10320">
                    <name>Documented</name>
                                                                <inwardlinks description="is documented by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="1455241">KAFKA-149</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                                                                                                                                                                        <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[5002K00000e6tEqQAI, 5002K00000oegDeQAI]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10257" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Documentation Changes</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10250"><![CDATA[Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_14266" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Documentation Changes Summary</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;Added support for the following settings:&lt;/p&gt;

&lt;p&gt;errors.tolerance = &lt;span class=&quot;error&quot;&gt;&amp;#91;none, all&amp;#93;&lt;/span&gt; &lt;br/&gt;
Errors tolerance controls: The behavior for tolerating errors during connector operation. &apos;none&apos; is the default value and signals that any error will result in an immediate connector task failure; &apos;all&apos; &quot;changes the behavior to skip over problematic records.&quot;;&lt;/p&gt;

&lt;p&gt;errors.log.enable=&lt;span class=&quot;error&quot;&gt;&amp;#91;true,false&amp;#93;&lt;/span&gt; &lt;br/&gt;
If true, write each error and the details of the failed operation and problematic record to the Connect application log. This is &apos;false&apos; by default, so that only errors that are not tolerated are reported.&lt;/p&gt;

&lt;p&gt;errors.deadletterqueue.topic.name=&quot;someTopic&quot;&lt;br/&gt;
Whether to output conversion errors to the dead letter queue. Stops poison messages when using schemas, any message will be outputted as extended json on the specified topic. By default messages are not outputted to the dead letter queue. Also requires `errors.tolerance=all`&lt;/p&gt;
</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>KAFKA-134</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hw338v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                </customfields>
    </item>
</channel>
</rss>