<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:44:08 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-17357] Add flag to enable logging in JSON format</title>
                <link>https://jira.mongodb.org/browse/SERVER-17357</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The mongo logs have a lot of varied information and, as a result, lack a consistent structure. This makes writing accurate and efficient parsers difficult because your parser has to understand how the line will be printed in each context. For instance, queries are logged very differently from updates:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Tue Feb 17 21:19:53.300 [conn947952] update foo.bar query: { _id: &quot;abc&quot; } update: { $set: { x: 0 } } nscanned:1 nscannedObjects:1 nMatched:1 nModified:1 keyUpdates:0 numYields:0 locks(micros) w:168 0ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;vs.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Tue Feb 17 21:19:53.301 [conn927275] query foo.bar query: { $query: { _id: &quot;abc&quot; } } planSummary: COLLSCAN ntoreturn:100 ntoskip:0 nscanned:1 nscannedObjects:1 keyUpdates:0 numYields:0 locks(micros) r:93 nreturned:1 reslen:521 0ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Outside of the realm of op queries, the lines get even more diverse:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Fri Feb 20 21:56:12.332 [initandlisten] connection accepted from 127.0.0.1:48345 #6552246 (5768 connections now open)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Fri Feb 20 21:57:02.003 [conn6479488] getmore local.oplog.rs cursorid:901395238872875 ntoreturn:0 keyUpdates:0 numYields:0 locks(micros) r:40 nreturned:1 reslen:201 4ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Fri Feb 20 21:57:42.989 [conn6549710] Plan 2 involved in multi-way tie. ns: data.foo query: { foo: &quot;bar&quot; } sort: {} projection: {} skip: 0 limit: 0 score: 1.5003 summary: IXSCAN { foo: 1 }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;This type of data seems like a good candidate for JSON formatting. It would be helpful to have a config option for having all log lines printed in JSON. This would make parsing straightforward and consistent regardless of the presence or absence of various fields.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;time&quot;: &quot;Tue Feb 17 21:19:53.301&quot;, &quot;thread&quot;: &quot;conn927275&quot;, &quot;op&quot;: &quot;query&quot;, &quot;ns&quot;: &quot;foo.bar&quot;, &quot;query&quot;: &quot;{ $query: { _id: \&quot;abc\&quot; } }&quot;, &quot;planSummary&quot;: &quot;COLLSCAN&quot;, &quot;ntoreturn&quot;:100, &quot;ntoskip&quot;:0, &quot;nscanned&quot;:1, &quot;nscannedObjects&quot;:1, &quot;keyUpdates&quot;:0, &quot;numYields&quot;:0, &quot;locks(micros)&quot; : { &quot;r&quot;:93 }, &quot;nreturned&quot;:1, &quot;reslen&quot;:521, &quot;duration_ms&quot;:0 }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="185806">SERVER-17357</key>
            <summary>Add flag to enable logging in JSON format</summary>
                <type id="2" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14711&amp;avatarType=issuetype">New Feature</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="backlog-server-devtools">DO NOT USE - Backlog - Dev Tools</assignee>
                                    <reporter username="tredman@fb.com">Travis Redman</reporter>
                        <labels>
                    </labels>
                <created>Mon, 23 Feb 2015 23:00:21 +0000</created>
                <updated>Tue, 6 Dec 2022 19:10:06 +0000</updated>
                            <resolved>Wed, 8 Apr 2020 18:14:25 +0000</resolved>
                                                    <fixVersion>4.4.0</fixVersion>
                                    <component>Logging</component>
                                        <votes>33</votes>
                                    <watches>41</watches>
                                                                                                                <comments>
                            <comment id="2838270" author="bcj6483@gmail.com" created="Fri, 7 Feb 2020 15:29:03 +0000"  >&lt;p&gt;If changing things from the inside out is so difficult can you add a grok parser that can at least make sense of the logs?&#160; I understand changing everything to JSON is tough, however, if you tried to parse your own logs you would say wow these are really non standard and vary wildly from log to log.&#160; Then you could make some changes to your logs to make them more uniform and maybe this need goes away as they are simple to parse.&lt;/p&gt;</comment>
                            <comment id="2818300" author="david.lambert@sabre.com" created="Thu, 6 Feb 2020 22:01:02 +0000"  >&lt;p&gt;So, this has moved back to &quot;Backlog&quot;?&#160; Was really looking forward to it being release in 4.3... Do you have an updated release date?&lt;/p&gt;</comment>
                            <comment id="2564742" author="lcharbon@ualberta.ca" created="Mon, 25 Nov 2019 16:37:20 +0000"  >&lt;p&gt;Side note: Would be nice if JSON logs can be streamed remotely through Kafka or other means.&lt;/p&gt;</comment>
                            <comment id="2209799" author="acm" created="Thu, 11 Apr 2019 16:00:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.lambert%40sabre.com&quot; class=&quot;user-hover&quot; rel=&quot;david.lambert@sabre.com&quot;&gt;david.lambert@sabre.com&lt;/a&gt; - &lt;/p&gt;

&lt;p&gt;Thanks for letting us know that this issue is important to you. My team is actually in the process of planning our upcoming work for the next release, and a project to address this issue is among the items we are considering. As I&apos;m sure you can imagine though, there are many potential projects competing for finite development resources.&lt;/p&gt;

&lt;p&gt;If we do end up pursuing the project, it will not be a simple change. The issue is that almost all logging in the server codebase is written in the following style:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;log() &amp;lt;&amp;lt; &quot;Foo has the value&quot; &amp;lt;&amp;lt; foo;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Here, the object returned by the call to &lt;tt&gt;log()&lt;/tt&gt; is effectively a C++ &lt;tt&gt;std::ostream&lt;/tt&gt;, and the &lt;tt&gt;&amp;lt;&amp;lt;&lt;/tt&gt; operators are the standard C++ streaming operators, which incrementally builds a &lt;tt&gt;std::string&lt;/tt&gt;, and then logs that generated &lt;tt&gt;std::string&lt;/tt&gt; as a single item. And there are thousands of such calls.&lt;/p&gt;

&lt;p&gt;In order to log to other formats like JSON, this needs to change, since once we have stringified and concatenated the arguments it is too late to change the format. Non-exhaustively, and for the purpose of illustration, here are three approaches we might take to change that. We could:&lt;/p&gt;

&lt;p&gt;1) Manually re-write all of the logging statements in the codebase to target a wholly new logging framework that is not based on streams.&lt;br/&gt;
2) Use a mechanism like C++ expression templates to defer evaluation of all arguments to &lt;tt&gt;&amp;lt;&amp;lt;&lt;/tt&gt; in each log statement, instead accumulating them into a &quot;log statement arguments packet&quot; of some sort (presumably a &lt;tt&gt;std::variant&lt;/tt&gt;) and then defining different evaluators that can unroll the argument packet into different formats: a single &lt;tt&gt;std::string&lt;/tt&gt;, a JSON object, etc.&lt;br/&gt;
3) Implement the wholly new logging framework from 1, above, and then implement a source-to-source transformation engine using something like LLVMs AST level transformations to automatically re-write all existing log statements into the new syntax.&lt;/p&gt;

&lt;p&gt;Each of these approaches has its challenges (option one is probably a non-starter), so a proper design will need to be written, probably with some proof of concept work as well to help steer the design choices. It is also important that we ensure that changes to our logging system do not have adverse performance consequences, particularly when dealing with debug logging statements. Such statements are almost always disabled in production so they need to be close to zero overhead, especially in hot paths through the database.&lt;/p&gt;

&lt;p&gt;The various support and engineering teams at MongoDB also spend a lot of time reading and analyzing MongoDB log files. Making this change would not only benefit our customers, but would be invaluable for us as well, so there is genuine interest in making this change purely on the technical merits.&lt;/p&gt;

&lt;p&gt;I hope that helps clarify the state of this request. Any specific input or requests you might have regarding the implementation are most welcome. Thanks again for the helpful feedback!&lt;/p&gt;</comment>
                            <comment id="2209591" author="david.lambert@sabre.com" created="Thu, 11 Apr 2019 14:12:52 +0000"  >&lt;p&gt;I know my tiny input to this isn&apos;t going to make Mongo wake up and do the right thing, but here it is.&lt;/p&gt;

&lt;p&gt;I too have spent &lt;b&gt;WAY&lt;/b&gt; to much time writing a logstash/GROK parser for logs and it&apos;s ugly and doesn&apos;t catch everything.&#160; But it does get the major job done.&lt;/p&gt;

&lt;p&gt;The idea that a document database company can&apos;t write logs out in a document form is just crazy really.&#160; Would almost be willing to bet that they are stored internally in json.&lt;/p&gt;

&lt;p&gt;Sadly, unless they find a way to monetize this, they aren&apos;t going to do the right thing...&#160;&lt;/p&gt;</comment>
                            <comment id="2209309" author="petr.kukral@centrum.cz" created="Thu, 11 Apr 2019 08:49:12 +0000"  >&lt;p&gt;+1 for this. We are in age of OpenShift and EFK where we send logs of our applications in JSON format.&#160;This way is far more easy and managable then creating parsing rules for every specific type of logs.&lt;/p&gt;</comment>
                            <comment id="2064772" author="nwhitehead" created="Fri, 16 Nov 2018 17:59:38 +0000"  >&lt;p&gt;If this issue moves, please add json logging to mongos as well.&lt;/p&gt;</comment>
                            <comment id="1809359" author="masterxilo" created="Mon, 19 Feb 2018 12:36:57 +0000"  >&lt;p&gt;&lt;a href=&quot;https://github.com/mongodb-js/log&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb-js/log&lt;/a&gt; attempts to provide a parser.&lt;/p&gt;

&lt;p&gt;Some motivation to implement this:&lt;/p&gt;

&lt;p&gt;From&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://engineering.linkedin.com/distributed-systems/log-what-every-software-engineer-should-know-about-real-time-datas-unifying&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://engineering.linkedin.com/distributed-systems/log-what-every-software-engineer-should-know-about-real-time-datas-unifying&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;(Actually, if you think about it, the idea of humans reading through logs on individual machines is something of an anachronism. This approach quickly becomes an unmanageable strategy when many services and servers are involved and the purpose of logs quickly becomes as an input to queries and graphs to understand behavior across many machines&#8212;something for which english text in files is not nearly as appropriate as the kind structured log described here.)&lt;/p&gt;&lt;/blockquote&gt;</comment>
                            <comment id="1797149" author="kaymakchi" created="Tue, 6 Feb 2018 19:22:15 +0000"  >&lt;p&gt;Any news on this task? &lt;/p&gt;</comment>
                            <comment id="1729829" author="mzraly" created="Sun, 19 Nov 2017 15:20:22 +0000"  >&lt;p&gt;Agreed.  I have spent those hours parsing MongoDB logs and it is a frustrating experience, both because of the time I had to spend dealing with the crazy format (query too long to print? we&apos;ll overwrite the middle with a 10-period ellipsis and break parseability!) and the time my parsing code has to spend working around it.&lt;/p&gt;

&lt;p&gt;Why parse logs at all you ask?&lt;/p&gt;

&lt;p&gt;First, to get detailed collection- and query-level data without incurring the expense of enabling the profiler.  I can set the slow query threshold to 0 if I want to look at everything, and that&apos;s less of an issue to me than enabling the profiler because log files go to a separate disk so there is no contention for IOPS.&lt;/p&gt;

&lt;p&gt;Second, to get data that I can&apos;t get from serverStatus or collStats commands.  For example, which collections or queries are seeing lock contention (acquireWaitCount, timeAcquiringMicros)?  That data is only provided at the server level in the serverStatus locks document.&lt;/p&gt;

&lt;p&gt;It&apos;s also kind of ironic that everything else in MongoDB is so JSON-centric, but not the logs.&lt;/p&gt;

&lt;p&gt;Besides, think about how much easier it would be for you guys to implement per-query analytics in Ops Manager or Atlas if you had an easy-to-parse log format that you could extend without breaking parsing code?  Isn&apos;t that the promise of using a document format like JSON in the database to begin with?&lt;/p&gt;

&lt;p&gt;You could then take things one step further and imagine a downstream application that takes the JSON-formatted logs and writes them, or a subset of them, to a capped collection in another MongoDB installation where they can be easily queried.&lt;/p&gt;

&lt;p&gt;Maybe you could use this to implement something like AWS RDS Performance Insights (&lt;a href=&quot;https://aws.amazon.com/rds/performance-insights/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://aws.amazon.com/rds/performance-insights/&lt;/a&gt;) without having to enable the profiler.&lt;/p&gt;

&lt;p&gt;It&apos;s all goodness, really.&lt;/p&gt;
</comment>
                            <comment id="1729812" author="jsirex" created="Sun, 19 Nov 2017 12:02:21 +0000"  >&lt;p&gt;Up!&lt;/p&gt;

&lt;p&gt;Either implement strict log format for everything or make it possible to change format type, like make each log entry in json.&lt;br/&gt;
It is impossible to parse your logs: sudden warning in the middle of message, truncation, different formats of message.&lt;br/&gt;
Even your documentation is lying:&lt;br/&gt;
&lt;a href=&quot;https://docs.mongodb.com/manual/reference/log-messages/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://docs.mongodb.com/manual/reference/log-messages/&lt;/a&gt;&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&amp;lt;timestamp&amp;gt; &amp;lt;severity&amp;gt; &amp;lt;component&amp;gt; [&amp;lt;context&amp;gt;] &amp;lt;message&amp;gt;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;Here &amp;lt;component&amp;gt; field has padding, so I must read it like:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&amp;lt;timestamp&amp;gt; &amp;lt;severity&amp;gt; &amp;lt;component&amp;gt; +[&amp;lt;context&amp;gt;] (warning:.* \.\.\. )?&amp;lt;message&amp;gt;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Because of crappy logs we have:&lt;br/&gt;
&lt;a href=&quot;https://github.com/rueckstiess/mongodb-log-spec&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/rueckstiess/mongodb-log-spec&lt;/a&gt; - specification???&lt;br/&gt;
&lt;a href=&quot;https://github.com/vjeantet/grok/blob/master/patterns/mongodb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/vjeantet/grok/blob/master/patterns/mongodb&lt;/a&gt; - wrong&lt;br/&gt;
&lt;a href=&quot;https://gist.github.com/gswallow/9011322&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://gist.github.com/gswallow/9011322&lt;/a&gt; - wrong&lt;br/&gt;
&lt;a href=&quot;https://gist.github.com/xizhibei/081444ea66dbf527624b4d53b635f9bd&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://gist.github.com/xizhibei/081444ea66dbf527624b4d53b635f9bd&lt;/a&gt; - wrong&lt;br/&gt;
&lt;a href=&quot;https://gist.github.com/mohit/6207215&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://gist.github.com/mohit/6207215&lt;/a&gt; - wrong&lt;/p&gt;

&lt;p&gt;I have my own grok. And it is huge, it is ugly, it doesn&apos;t work well.&lt;br/&gt;
We have mongodb clusters, we need to aggregate logs for different type of analysis:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Time spent of query&lt;/li&gt;
	&lt;li&gt;Scanned object&lt;/li&gt;
	&lt;li&gt;Lock statistics&lt;/li&gt;
	&lt;li&gt;Database name&lt;/li&gt;
	&lt;li&gt;Collection name&lt;/li&gt;
	&lt;li&gt;etc&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;You have to spend couple of hours to implement json or some kind of format where I can exactly extract all fields.&lt;br/&gt;
On other hand thousands of mongodb users MUST waste tens hours to implement something partially working.&lt;/p&gt;</comment>
                            <comment id="1597830" author="jsirex" created="Thu, 15 Jun 2017 12:25:26 +0000"  >&lt;p&gt;The same issue: It is impossible to work with mongodb logs. One more case when you have multiple lines and garbage with long query in logs like:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-06-15T13:49:07.103+0300 I WRITE    [conn4604436] warning: log line attempted (15kB) over max size (10kB), printing beginning and end ... update stag_peopl LONG LONG QUERY GOES HERE, AND FIELD=&quot;multi&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;lines&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;of text&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&quot;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1535133" author="2k.pravin@gmail.com" created="Tue, 28 Mar 2017 18:19:11 +0000"  >&lt;p&gt;Now a days JSON is being supported by almost all the applications/parser. I want to use ELK stack for our log monitoring. Could you please let us know if you have anything on pipeline to provide Log in JSON format?&lt;/p&gt;</comment>
                            <comment id="1111739" author="abcfy2" created="Fri, 11 Dec 2015 05:42:16 +0000"  >&lt;p&gt;+1 For this feature. Mongodb log is hard to parse. Logstash grok pattern could not do well with mongodb log.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="243421">SERVER-21857</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25145"><![CDATA[Developer Tools]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[5002K00000ll4EtQAI]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 24 Feb 2015 00:24:39 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/PM-1234'>PM-1234</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>alexander.golin@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>andrew.morrow@mongodb.com</customfieldvalue>
            <customfieldvalue>Kaymakchi</customfieldvalue>
            <customfieldvalue>bcj6483@gmail.com</customfieldvalue>
            <customfieldvalue>david.lambert@sabre.com</customfieldvalue>
            <customfieldvalue>backlog-server-devtools</customfieldvalue>
            <customfieldvalue>abcfy2</customfieldvalue>
            <customfieldvalue>lcharbon@ualberta.ca</customfieldvalue>
            <customfieldvalue>mzraly</customfieldvalue>
            <customfieldvalue>nwhitehead</customfieldvalue>
            <customfieldvalue>masterxilo</customfieldvalue>
            <customfieldvalue>petr.kukral@centrum.cz</customfieldvalue>
            <customfieldvalue>2k.pravin@gmail.com</customfieldvalue>
            <customfieldvalue>tredman@fb.com</customfieldvalue>
            <customfieldvalue>jsirex</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrkxm7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr9jen:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>163549</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hsg8kf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>