<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:11:24 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-46406] Evaluate JSON logging performance in test suites</title>
                <link>https://jira.mongodb.org/browse/SERVER-46406</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Use evergreen data to compare logging performance and evaluate any opportunities for performance improvements.&lt;/p&gt;

&lt;p&gt;One suggestion has been to disable UTF-8 validation in logging. Other possibilities include examining memory and I/O overhead.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1185044">SERVER-46406</key>
            <summary>Evaluate JSON logging performance in test suites</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="henrik.edin@mongodb.com">Henrik Edin</assignee>
                                    <reporter username="mark.benvenuto@mongodb.com">Mark Benvenuto</reporter>
                        <labels>
                    </labels>
                <created>Tue, 25 Feb 2020 21:07:52 +0000</created>
                <updated>Sun, 29 Oct 2023 22:11:42 +0000</updated>
                            <resolved>Fri, 13 Mar 2020 17:18:30 +0000</resolved>
                                                    <fixVersion>4.4.0-rc0</fixVersion>
                    <fixVersion>4.7.0</fixVersion>
                                    <component>Logging</component>
                                        <votes>0</votes>
                                    <watches>2</watches>
                                                                                                                <comments>
                            <comment id="2982212" author="xgen-internal-githook" created="Mon, 16 Mar 2020 20:31:05 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;henrikedin&apos;, &apos;name&apos;: &apos;Henrik Edin&apos;, &apos;email&apos;: &apos;henrik.edin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46406&quot; title=&quot;Evaluate JSON logging performance in test suites&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46406&quot;&gt;&lt;del&gt;SERVER-46406&lt;/del&gt;&lt;/a&gt; Perf improvements in the logger.&lt;/p&gt;

&lt;p&gt;Use fmt::compile and fmt::format_int when possible.&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 5094b4f2f2ee9de07dc4c3d3c320e967595496f1)&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/bc43f398697282a5fedb129d6913ef09244af6fb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/bc43f398697282a5fedb129d6913ef09244af6fb&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2978673" author="henrik.edin" created="Fri, 13 Mar 2020 15:06:41 +0000"  >&lt;p&gt;Looked into the difference in run time by comparing the sharding suite in these two runs:&lt;/p&gt;

&lt;p&gt;Jan 24:&lt;br/&gt;
&lt;a href=&quot;https://evergreen.mongodb.com/task/mongodb_mongo_master_enterprise_rhel_62_64_bit_display_sharding_38ec223f7478be14fc3bf082643a1109efaeb57c_20_01_25_02_28_44&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://evergreen.mongodb.com/task/mongodb_mongo_master_enterprise_rhel_62_64_bit_display_sharding_38ec223f7478be14fc3bf082643a1109efaeb57c_20_01_25_02_28_44&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Feb 25:&lt;br/&gt;
&lt;a href=&quot;https://evergreen.mongodb.com/task/mongodb_mongo_master_enterprise_rhel_62_64_bit_display_sharding_742ac9b37b1d8f489e1b259a0a3575f8811edac4_20_02_25_14_46_48&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://evergreen.mongodb.com/task/mongodb_mongo_master_enterprise_rhel_62_64_bit_display_sharding_742ac9b37b1d8f489e1b259a0a3575f8811edac4_20_02_25_14_46_48&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Execution subtasks went from 26 to 28 &lt;br/&gt;
Cumulative time went from &lt;b&gt;1h 54m 55s&lt;/b&gt; to &lt;b&gt;2h 16m 53s&lt;/b&gt;&lt;br/&gt;
Wall clock time went from &lt;b&gt;31m 32s&lt;/b&gt; to &lt;b&gt;1h 0m 10s&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;Before there was &lt;b&gt;972&lt;/b&gt; jobs that had a total runtime of &lt;b&gt;3h 37m 15s&lt;/b&gt;&lt;br/&gt;
After there was &lt;b&gt;1012&lt;/b&gt; jobs that had a total runtime of &lt;b&gt;3h 58m 15s&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;The number of jobs increased, the cumulative time increased with around 20% but the wall clock time with a whopping 100%.&lt;/p&gt;

&lt;p&gt;When looking at the logs from the various subtasks you notice:&lt;/p&gt;

&lt;p&gt;Before the gen task finished &lt;b&gt;21:33&lt;/b&gt; and the exec tasks started around &lt;b&gt;21:53&lt;/b&gt; and ran for up to 10min&lt;br/&gt;
But after we have a gen task that finished &lt;b&gt;10:01&lt;/b&gt; and some exec tasks that start at &lt;b&gt;10:20&lt;/b&gt; but most of them doesn&apos;t start until &lt;b&gt;10:50&lt;/b&gt;. That&apos;s &lt;b&gt;50min&lt;/b&gt; of no work, this is where the wall clock time is going.&lt;/p&gt;

&lt;p&gt;So from what I can tell the problem is a scheduling / resource issue at Evergreen. There&apos;s not enough hosts to schedule the execution, leading to long wall clock times. When they do eventually run the execution time is not much different and even if there was an increased total execution time there are also more tests that run now than before.&lt;/p&gt;

&lt;p&gt;I will close this ticket with the CPU perf improvements that has been committed.&lt;/p&gt;</comment>
                            <comment id="2978566" author="xgen-internal-githook" created="Fri, 13 Mar 2020 14:36:56 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Henrik Edin&apos;, &apos;username&apos;: &apos;henrikedin&apos;, &apos;email&apos;: &apos;henrik.edin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46406&quot; title=&quot;Evaluate JSON logging performance in test suites&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46406&quot;&gt;&lt;del&gt;SERVER-46406&lt;/del&gt;&lt;/a&gt; Perf improvements in the logger.&lt;/p&gt;

&lt;p&gt;Use fmt::compile and fmt::format_int when possible.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/5094b4f2f2ee9de07dc4c3d3c320e967595496f1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/5094b4f2f2ee9de07dc4c3d3c320e967595496f1&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2975831" author="mark.benvenuto" created="Thu, 12 Mar 2020 18:19:51 +0000"  >&lt;p&gt;Assigning to &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=henrik.edin&quot; class=&quot;user-hover&quot; rel=&quot;henrik.edin&quot;&gt;henrik.edin&lt;/a&gt; since he is making the code changes to address some of the issues identified.&lt;/p&gt;</comment>
                            <comment id="2968082" author="bruce.lucas@10gen.com" created="Wed, 11 Mar 2020 16:55:22 +0000"  >&lt;p&gt;Thanks Henrik, that&apos;s good to know.&lt;/p&gt;</comment>
                            <comment id="2966864" author="henrik.edin" created="Wed, 11 Mar 2020 15:28:08 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=mark.benvenuto&quot; class=&quot;user-hover&quot; rel=&quot;mark.benvenuto&quot;&gt;mark.benvenuto&lt;/a&gt; I ran the same benchmark on 4.2 and it shows a similar picture. LogstreamBuilder::~LogstreamBuilder() + OpDebug::report() that&apos;s used to log in completeAndLogOperation() is almost as much time as mongo::performUpdates(). That might be the bulk of the logging, but formatting in other log statements using operator&amp;lt;&amp;lt; is not included because it happens outside of the log system in the old logger.&lt;/p&gt;</comment>
                            <comment id="2961273" author="henrik.edin" created="Tue, 10 Mar 2020 17:36:54 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=mark.benvenuto&quot; class=&quot;user-hover&quot; rel=&quot;mark.benvenuto&quot;&gt;mark.benvenuto&lt;/a&gt; Sent you a CR for low hanging fruit improvements.&lt;/p&gt;

&lt;p&gt;V1 escaped in BSONObj::jsonString, but it did not escape all the strings as we do now. And when BSONObj was logged with the regular toString() there is no escaping.&lt;/p&gt;

&lt;p&gt;Using SSE to further improve escapeForJSON is an interesting idea. But I think that&apos;s too large to do as part of this. We should ticket it out separatley.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="2955611" author="mark.benvenuto" created="Mon, 9 Mar 2020 21:02:21 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;, I did not compare against the original version. I originally thought of comparing a &quot;verbose&quot; jstest test but there is more variance for any given test to be a good comparison. I think it would be a good idea for compare YSCB on say 4.2.x which is still using the original V1 logging framework. It would be interesting to look at the relative cost comparison.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=henrik.edin&quot; class=&quot;user-hover&quot; rel=&quot;henrik.edin&quot;&gt;henrik.edin&lt;/a&gt;&lt;br/&gt;
1. itoa - We have are own optimized version which is even faster - see itoa.h&lt;br/&gt;
2. I see how it buffers now. The code was difficult for me to read. I still believe we have an opportunity to optimize this code.&lt;/p&gt;

&lt;p&gt;Where does the log V1 code escape? I found some code on Windows for newlines and some UTF-8 stuff for the Windows console. I did not find any code for the escaping when writing to a file.&lt;/p&gt;

&lt;p&gt;I think the Evergreen per problems are two fold:&lt;br/&gt;
1. Larger I/O - as has been discussed, we are 20-50% larger. This is probably most of the cost.&lt;br/&gt;
2. CPU - CPU time matters which is causing some of the slow down.&lt;/p&gt;

&lt;p&gt;In general, benchmarks are designed to not log which is why they will not show a problem. My test was artificially trying to push the logging code harder than a benchmark should.&lt;/p&gt;

&lt;p&gt;&lt;ins&gt;Appendix&lt;/ins&gt;:&lt;br/&gt;
For an example of optimizations, we could focus on just scanning for valid ascii. We could do various tricks to scan blocks of characters instead of one at a time. We could do this with regular CPU instructions or probably using SIMD in many cases.&lt;br/&gt;
There is also the magic &lt;a href=&quot;https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestri&amp;amp;expand=786,834&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;_mm_cmpestri&lt;/a&gt; SSE 4.2 instruction.&lt;/p&gt;

&lt;p&gt;Untested example code:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;// https://graphics.stanford.edu/~seander/bithacks.html&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#define haszero(v) (((v) - 0x01010101UL) &amp;amp; ~(v) &amp;amp; 0x80808080UL)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#define hasvalue(x,n) (haszero((x) ^ (~0UL/255 * (n))))&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#define hasless(x,n) (((x)-~0UL/255*(n))&amp;amp;~(x)&amp;amp;~0UL/255*128)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;int utf8_mask = 0x80808080;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;bool isValidJSONCharacters(char* str, int len) {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    while (len &amp;gt; 4) {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        int* word_ptr = reinterpret_cast&amp;lt;int*&amp;gt;(str);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        for(int i = 0; i &amp;lt; (len /4); i++) {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            int word = word_ptr[i];&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            if ((word &amp;amp; utf8_mask) != 0 || hasless(word,0x1f) || hasvalue(word,&apos; &apos;) || hasvalue(word,&apos;\\&apos;) || hasvalue(word,0x7f) ) {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;                return false;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        str += 4 * (len /4);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        len -= 4 * (len /4);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    for(int i = 0; i &amp;lt; len; i++) {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        if( str[i] &amp;lt; 0x1f || str[i] == &apos; &apos; || str[i] == &apos;\\&apos; || str[i] == 0x7f) {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            return false;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    return true;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="2950797" author="henrik.edin" created="Sat, 7 Mar 2020 17:39:46 +0000"  >&lt;p&gt;Thanks for the excellent analysis Mark! A few comments and clarifications:&lt;/p&gt;

&lt;p&gt;1. Using a faster formatting for integers than fmt::format is probably a good idea. Perhaps the answer is itoa, but there&apos;s also a fast path for integers that we can use in libfmt called format_int that&apos;s supposed to be very fast. See this benchmark: &lt;a href=&quot;http://www.zverovich.net/2013/09/07/integer-to-string-conversion-in-cplusplus.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.zverovich.net/2013/09/07/integer-to-string-conversion-in-cplusplus.html&lt;/a&gt; This seem to be a low hanging fruit we should certainly pursue.&lt;/p&gt;

&lt;p&gt;2. escapeForJSON does not write bytes one at a time, but it analyze them one at a time. At a minimum this is something we need to do to escape ASCII control characters (the old logger also did this) but in addition it also validates the UTF-8 encoding by doing bit masking. Blocks of data that can be written unchanged are written as blocks.&lt;/p&gt;

&lt;p&gt;The overall time spent in the logger looks worrisome, lots of it is spent in boost::log and there&apos;s not much we can do here apart from modify the library or use a different logging backend.&lt;/p&gt;

&lt;p&gt;It would be interesting to run the same benchmark on 4.2 and see how much time we spent in the old logger. It was also doing expensive formatting using iostreams and might also show up high up in this benchmark. The perf fallout so far from this project is one test being slower but a couple being faster.&lt;/p&gt;

&lt;p&gt;Do you think the CPU perf is what&apos;s causing issues on Evergreen? I would think that it would affect many more benchmarks if that was the case.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="2950783" author="bruce.lucas@10gen.com" created="Sat, 7 Mar 2020 17:33:53 +0000"  >&lt;p&gt;Great work Mark. Do you have an idea how the cost of logging compares with v1? Will customers who are experiencing a performance problem causing a lot of slow query logging, or who intentionally set slowms to 0 or set log level to 1, see it more impacted by logging than before?&lt;/p&gt;</comment>
                            <comment id="2949786" author="mark.benvenuto" created="Fri, 6 Mar 2020 23:34:36 +0000"  >&lt;p&gt;I did two analyses:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;Verified I/O was reasonably efficient.&lt;/li&gt;
	&lt;li&gt;Did sampling based profile of logging code&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&lt;ins&gt;I/O&lt;/ins&gt;&lt;/p&gt;

&lt;p&gt;We are flushing whole lines. We are not writing a single log line over multiple calls. The write is occurring with ostream::flush().&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&lt;b&gt;&lt;ins&gt;Logging Code&lt;/ins&gt;&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;&lt;ins&gt;Test case&lt;/ins&gt;&lt;/p&gt;

&lt;p&gt;YCSB, 50000 Records, workload a&lt;/p&gt;

&lt;p&gt;MongoD args: --logpath=/dev/null --slowms=0 --storageEngine=inMemory --dbpath=/tmp/foo&lt;/p&gt;

&lt;p&gt;Profiler: VTune 2020&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&lt;ins&gt;Results:&lt;/ins&gt;&lt;/p&gt;

&lt;p&gt;If we look at the overall most expensive functions in this artificial workload to magnify the cost of logging, we see that logging is almost as expensive as doing updates and we spend about have the command invocation time in logging.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/251510/251510_Log_Overview.png&quot; height=&quot;251&quot; width=&quot;784&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Breaking it down: &lt;tt&gt;open_record&lt;/tt&gt;&#160;is unexpectedly expensive it does a hash table lookup just to determine if we need to log and takes two different read locks.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/251511/251511_open_record.png&quot; height=&quot;313&quot; width=&quot;931&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt; &#160;&lt;/p&gt;

&lt;p&gt;Next:&lt;/p&gt;

&lt;p&gt;As we move into the meat of the logging, our test case is mostly slow query logs which need to log BSON.&#160; As a result, we see how expensive things are for formatting BSONObj.&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;writeString - (i.e. really&#160;escapeForJSON) - we need to avoid writing one character at a time. It should be optimized for ASCII at a small performance penalty for non-ascii.&lt;/li&gt;
	&lt;li&gt;format_to - it is formatting a uint64_t integer and yet dominates the cost of formatting over a string.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/251512/251512_json_formatter.png&quot; height=&quot;358&quot; width=&quot;988&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt; &#160;&lt;/p&gt;

&lt;p&gt;Finally:&lt;/p&gt;

&lt;p&gt;In the rest of the formatting code, we see that libfmt dominates the cost excessively. Most of this cost comes from the library having to parse the format string. Also, outputDateAsISOStringLocal is expensive because of a mix of tz_convert and sprintf.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/251513/251513_libfmt_slow.png&quot; height=&quot;436&quot; width=&quot;920&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;And no, the dominant cost for libfmt is not coming from it appending the values to the output stream. It is from parsing the format specifiers.&lt;/p&gt;

&lt;p&gt;&lt;ins&gt;Recommendations&lt;/ins&gt;:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;Remove libfmt from the main logging code path. libfmt is just a new version of sprintf and sprintf is slow. We should call functions like itoa instead and do more manual formatting.&lt;/li&gt;
	&lt;li&gt;Optimize&#160;escapeForJSON to stop appending one character at a time. It should be optimized for ASCII over UTF-8 and consider escaping an unlikely event.&lt;/li&gt;
	&lt;li&gt;Improve outputDateAsISOStingLocal by avoiding sprintf and family.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;&#160;&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="1198118">SERVER-46467</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="251510" name="Log_Overview.png" size="81126" author="mark.benvenuto@mongodb.com" created="Fri, 6 Mar 2020 23:17:58 +0000"/>
                            <attachment id="251512" name="json_formatter.png" size="162834" author="mark.benvenuto@mongodb.com" created="Fri, 6 Mar 2020 23:17:58 +0000"/>
                            <attachment id="251513" name="libfmt_slow.png" size="221055" author="mark.benvenuto@mongodb.com" created="Fri, 6 Mar 2020 23:17:58 +0000"/>
                            <attachment id="251511" name="open_record.png" size="133188" author="mark.benvenuto@mongodb.com" created="Fri, 6 Mar 2020 23:17:58 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>11.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 5 Mar 2020 16:12:39 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 47 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1234</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 47 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>henrik.edin@mongodb.com</customfieldvalue>
            <customfieldvalue>mark.benvenuto@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwutsf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hwiref:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3660">Security 2020-03-09</customfieldvalue>
    <customfieldvalue id="3737">Security 2020-03-23</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwug1r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>