Loading...

Type: Bug
Resolution: Duplicate
Priority: Major - P3
Fix Version/s: None
Affects Version/s: None
Component/s: None
Labels:
None

Operating System:
ALL
Sprint:
Security 2021-08-09

The python team found a server segmentation fault when starting a standalone for OCSP testing:

 [2021/06/28 19:38:06.552] db version v4.4.3
 [2021/06/28 19:38:06.552] Build Info: {
 [2021/06/28 19:38:06.552]     "version": "4.4.3",
 [2021/06/28 19:38:06.552]     "gitVersion": "913d6b62acfbb344dde1b116f4161360acd8fd13",
 [2021/06/28 19:38:06.552]     "openSSLVersion": "OpenSSL 1.0.2k-fips  26 Jan 2017",
 [2021/06/28 19:38:06.552]     "modules": [
 [2021/06/28 19:38:06.552]         "enterprise"
 [2021/06/28 19:38:06.552]     ],
 [2021/06/28 19:38:06.552]     "allocator": "tcmalloc",
 [2021/06/28 19:38:06.552]     "environment": {
 [2021/06/28 19:38:06.552]         "distmod": "amzn64",
 [2021/06/28 19:38:06.552]         "distarch": "x86_64",
 [2021/06/28 19:38:06.552]         "target_arch": "x86_64"
 [2021/06/28 19:38:06.552]     }
 [2021/06/28 19:38:06.552] }
...
 [2021/06/28 19:38:23.441] 2021-06-28 19:38:23,437 [ERROR] mongo_orchestration.servers:371 - Could not start Server. Please find server log below.
 [2021/06/28 19:38:23.441] =====================================================
 [2021/06/28 19:38:23.441] 2021-06-28 19:38:23,437 [ERROR] mongo_orchestration.servers:374 - {"t":{"$date":"2021-06-28T19:38:23.359+00:00"},"s":"D1", "c":"NETWORK",  "id":23214,   "ctx":"main","msg":"Client certificate name","attr":{"name":"L=OCSP-1,ST=NY,C=US,CN=localhost,OU=Kernel,O=MongoDB"}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.359+00:00"},"s":"D1", "c":"NETWORK",  "id":23215,   "ctx":"main","msg":"Server certificate name","attr":{"name":"L=OCSP-1,ST=NY,C=US,CN=localhost,OU=Kernel,O=MongoDB"}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.359+00:00"},"s":"D1", "c":"NETWORK",  "id":23216,   "ctx":"main","msg":"Server certificate expiration","attr":{"expiration":{"$date":"2040-03-21T15:56:22.000Z"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.360+00:00"},"s":"I",  "c":"CONTROL",  "id":23285,   "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.361+00:00"},"s":"W",  "c":"ASIO",     "id":22601,   "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.361+00:00"},"s":"I",  "c":"NETWORK",  "id":4648602, "ctx":"main","msg":"Implicit TCP FastOpen in use."}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.364+00:00"},"s":"W",  "c":"ASIO",     "id":22601,   "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.364+00:00"},"s":"D1", "c":"NETWORK",  "id":22940,   "ctx":"main","msg":"file descriptor and connection resource limits","attr":{"hard":64000,"soft":64000,"conn":51200}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.365+00:00"},"s":"I",  "c":"STORAGE",  "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":6689,"port":27017,"dbPath":"/data/mci/c0b78ba833ba55290c706232320e0caf/drivers-tools/.evergreen/orchestration/db/mongod-kvz26hy5","architecture":"64-bit","host":"ip-10-122-78-230"}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.365+00:00"},"s":"I",  "c":"CONTROL",  "id":23403,   "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.3","gitVersion":"913d6b62acfbb344dde1b116f4161360acd8fd13","openSSLVersion":"OpenSSL 1.0.2k-fips  26 Jan 2017","modules":["enterprise"],"allocator":"tcmalloc","environment":{"distmod":"amzn64","distarch":"x86_64","target_arch":"x86_64"}}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.365+00:00"},"s":"I",  "c":"CONTROL",  "id":51765,   "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Amazon Linux AMI release 2018.03","version":"Kernel 4.14.77-70.59.amzn1.x86_64"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.365+00:00"},"s":"I",  "c":"CONTROL",  "id":21951,   "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/data/mci/c0b78ba833ba55290c706232320e0caf/drivers-tools/.evergreen/orchestration/db/mongo-6ydd8ufb","net":{"bindIp":"127.0.0.1,::1","compression":{"compressors":"zstd,zlib,snappy,noop"},"ipv6":true,"port":27017,"tls":{"CAFile":"/data/mci/c0b78ba833ba55290c706232320e0caf/drivers-tools/.evergreen/ocsp/rsa/ca.pem","allowConnectionsWithoutCertificates":true,"allowInvalidCertificates":true,"certificateKeyFile":"/data/mci/c0b78ba833ba55290c706232320e0caf/drivers-tools/.evergreen/ocsp/rsa/server-mustStaple.pem","mode":"requireTLS"}},"replication":{"enableMajorityReadConcern":true,"oplogSizeMB":100},"setParameter":{"enableTestCommands":"1","maxTransactionLockRequestTimeoutMillis":"25","ocspEnabled":"true","periodicNoopIntervalSecs":"1","transactionLifetimeLimitSeconds":"3","writePeriodicNoops":"1"},"storage":{"dbPath":"/data/mci/c0b78ba833ba55290c706232320e0caf/drivers-tools/.evergreen/orchestration/db/mongod-kvz26hy5","journal":{"enabled":true}},"systemLog":{"destination":"file","logAppend":true,"path":"/data/mci/c0b78ba833ba55290c706232320e0caf/drivers-tools/.evergreen/orchestration/db/mongod-kvz26hy5/mongod.log","verbosity":1}}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.365+00:00"},"s":"D1", "c":"NETWORK",  "id":22940,   "ctx":"initandlisten","msg":"file descriptor and connection resource limits","attr":{"hard":64000,"soft":64000,"conn":51200}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.366+00:00"},"s":"D1", "c":"EXECUTOR", "id":23104,   "ctx":"OCSPManagerHTTP-0","msg":"Starting thread","attr":{"threadName":"OCSPManagerHTTP-0","poolName":"OCSPManagerHTTP"}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.367+00:00"},"s":"D1", "c":"EXECUTOR", "id":23104,   "ctx":"OCSPManagerHTTP-1","msg":"Starting thread","attr":{"threadName":"OCSPManagerHTTP-1","poolName":"OCSPManagerHTTP"}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.367+00:00"},"s":"F",  "c":"CONTROL",  "id":4757800, "ctx":"OCSPManagerHTTP-1","msg":"Writing fatal message","attr":{"message":"Invalid access at address: 0x18"}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.367+00:00"},"s":"F",  "c":"CONTROL",  "id":4757800, "ctx":"OCSPManagerHTTP-1","msg":"Writing fatal message","attr":{"message":"Got signal: 11 (Segmentation fault).\n"}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.368+00:00"},"s":"I",  "c":"STORAGE",  "id":22315,   "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=7508M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],"}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.369+00:00"},"s":"D1", "c":"-",        "id":23074,   "ctx":"OCSPManagerHTTP-0","msg":"User assertion","attr":{"error":"OperationFailed: Bad HTTP response from API server: Couldn't connect to server","file":"src/mongo/util/net/http_client_curl.cpp","line":344}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31431,   "ctx":"OCSPManagerHTTP-1","msg":"BACKTRACE: {bt}","attr":{"bt":{"backtrace":[{"a":"55B78CB3BA41","b":"55B789D16000","o":"2E25A41","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"},{"a":"55B78CB3D079","b":"55B789D16000","o":"2E27079","s":"_ZN5mongo15printStackTraceEv","s+":"29"},{"a":"55B78CB3AA1C","b":"55B789D16000","o":"2E24A1C","s":"_ZN5mongo12_GLOBAL__N_124abruptQuitWithAddrSignalEiP9siginfo_tPv","s+":"EC"},{"a":"7F961F360600","b":"7F961F351000","o":"F600","s":"_L_unlock_13","s+":"34"},{"a":"7F96207AD8FA","b":"7F962077A000","o":"338FA","s":"curl_multi_timeout","s+":"16A"},{"a":"7F96207C61CA","b":"7F962077A000","o":"4C1CA","s":"curl_slist_free_all","s+":"9D8A"},{"a":"7F96207AD14B","b":"7F962077A000","o":"3314B","s":"curl_multi_wait","s+":"BB"},{"a":"7F96207A6D70","b":"7F962077A000","o":"2CD70","s":"curl_easy_perform","s+":"F0"},{"a":"55B78CAA1F91","b":"55B789D16000","o":"2D8BF91","s":"_ZNK5mongo12_GLOBAL__N_114CurlHttpClient7requestENS_10HttpClient10HttpMethodENS_10StringDataENS_14ConstDataRangeE","s+":"2D1"},{"a":"55B78C652E37","b":"55B789D16000","o":"293CE37","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_11OCSPManager13requestStatusESt6vectorIhSaIhEENS_10StringDataENS_11OCSPPurposeEEUlT_E_EEDaOSB_EN12SpecificImpl4callEOS1_","s+":"C7"},{"a":"55B78C674C42","b":"55B789D16000","o":"295EC42","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockINS_12latch_detail5LatchEE","s+":"132"},{"a":"55B78C677466","b":"55B789D16000","o":"2961466","s":"_ZN5mongo10ThreadPool13_consumeTasksEv","s+":"86"},{"a":"55B78C678211","b":"55B789D16000","o":"2962211","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"E1"},{"a":"55B78C678540","b":"55B789D16000","o":"2962540","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_10ThreadPool25_startWorkerThread_inlockEvEUlvE2_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"60"},{"a":"55B78CCE076F","b":"55B789D16000","o":"2FCA76F","s":"execute_native_thread_routine","s+":"F"},{"a":"7F961F358E75","b":"7F961F351000","o":"7E75","s":"start_thread","s+":"C5"},{"a":"7F961F0818FD","b":"7F961EF83000","o":"FE8FD","s":"clone","s+":"6D"}],"processInfo":{"mongodbVersion":"4.4.3","gitVersion":"913d6b62acfbb344dde1b116f4161360acd8fd13","compiledModules":["enterprise"],"uname":{"sysname":"Linux","release":"4.14.77-70.59.amzn1.x86_64","version":"#1 SMP Mon Nov 12 22:02:45 UTC 2018","machine":"x86_64"},"somap":[{"b":"55B789D16000","elfType":3,"buildId":"F8C7004BDB4646F46541DFCC94D3999F274C288B"},{"b":"7F962077A000","path":"/usr/lib64/libcurl.so.4","elfType":3,"buildId":"CC3772AD47FA099DFDA2B50861CCD92FA719D101"},{"b":"7F961F351000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"383B229C0E6E99B4E3BA6FC8B8C096C103226984"},{"b":"7F961EF83000","path":"/lib64/libc.so.6","elfType":3,"buildId":"8BDBE5043577FC2EA218FAFD7EDF175D219698FB"}]}}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78CB3BA41","b":"55B789D16000","o":"2E25A41","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1E1"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78CB3D079","b":"55B789D16000","o":"2E27079","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78CB3AA1C","b":"55B789D16000","o":"2E24A1C","s":"_ZN5mongo12_GLOBAL__N_124abruptQuitWithAddrSignalEiP9siginfo_tPv","s+":"EC"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F961F360600","b":"7F961F351000","o":"F600","s":"_L_unlock_13","s+":"34"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F96207AD8FA","b":"7F962077A000","o":"338FA","s":"curl_multi_timeout","s+":"16A"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F96207C61CA","b":"7F962077A000","o":"4C1CA","s":"curl_slist_free_all","s+":"9D8A"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F96207AD14B","b":"7F962077A000","o":"3314B","s":"curl_multi_wait","s+":"BB"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F96207A6D70","b":"7F962077A000","o":"2CD70","s":"curl_easy_perform","s+":"F0"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78CAA1F91","b":"55B789D16000","o":"2D8BF91","s":"_ZNK5mongo12_GLOBAL__N_114CurlHttpClient7requestENS_10HttpClient10HttpMethodENS_10StringDataENS_14ConstDataRangeE","s+":"2D1"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78C652E37","b":"55B789D16000","o":"293CE37","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_11OCSPManager13requestStatusESt6vectorIhSaIhEENS_10StringDataENS_11OCSPPurposeEEUlT_E_EEDaOSB_EN12SpecificImpl4callEOS1_","s+":"C7"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78C674C42","b":"55B789D16000","o":"295EC42","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockINS_12latch_detail5LatchEE","s+":"132"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78C677466","b":"55B789D16000","o":"2961466","s":"_ZN5mongo10ThreadPool13_consumeTasksEv","s+":"86"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78C678211","b":"55B789D16000","o":"2962211","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"E1"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78C678540","b":"55B789D16000","o":"2962540","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_10ThreadPool25_startWorkerThread_inlockEvEUlvE2_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"60"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"55B78CCE076F","b":"55B789D16000","o":"2FCA76F","s":"execute_native_thread_routine","s+":"F"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F961F358E75","b":"7F961F351000","o":"7E75","s":"start_thread","s+":"C5"}}}
 [2021/06/28 19:38:23.441] {"t":{"$date":"2021-06-28T19:38:23.434+00:00"},"s":"I",  "c":"CONTROL",  "id":31427,   "ctx":"OCSPManagerHTTP-1","msg":"  Frame: {frame}","attr":{"frame":{"a":"7F961F0818FD","b":"7F961EF83000","o":"FE8FD","s":"clone","s+":"6D"}}}

https://spruce.mongodb.com/task/mongo_python_driver_3.12_ocsp_test__platform~awslinux_auth~noauth_ssl~ssl_mongodb_version~4.4_python_version~2.7_test_ocsp_rsa_delegate_valid_cert_server_staples_patch_866ed88e83534cd7d23dcb5bf879e353e25d7707_60da2225562343485eee9075_21_06_28_19_26_24/logs?execution=0

We restarted the task to see if the failure was consistent and the test completed normal on the second attempt.

The server is started with server-mustStaple.pem and the following config:

is related to

SERVER-54139 Remove CURL share support from HTTP curl client

Closed

Details

Description

Attachments

Issue Links

Activity

People

Dates