-
Type: Bug
-
Resolution: Done
-
Priority: Major - P3
-
None
-
Affects Version/s: None
-
Component/s: None
-
Labels:None
-
ALL
We have 3.2.x sharded cluster with authschema version 3 that we are trying to upgrade to the latest auth schema. However it appears to fail when we run the authSchema upgrade command on the shard routers.
mongos> db.system.version.findOne(\{"_id" : "authSchema"}); { "_id" : "authSchema", "currentVersion" : 3 } mongos> db.adminCommand(\{authSchemaUpgrade: 1 }); { "ok" : 0, "errmsg" : "field not found, expected type 2", "code" : 13111 }
Our config servers are still on mmapv1. Here is what we see in the logs on the first config server
2018-05-08T01:13:18.676+0000 I - [conn114] Assertion: 13111:field not found, expected type 2 2018-05-08T01:13:18.689+0000 I CONTROL [conn114] 0x1342b42 0x12de108 0x12c6448 0x12c64fc 0x9bccdb 0xb79737 0xb7d6c8 0xa0671b 0x9fa7ba 0x9fa602 0xb7681a 0xb7a1dd 0xb7a67f 0xb7a7a2 0xb8083b 0xbbace3 0xbbbb74 0xb11ff0 0xcd1d45 0x99ebcc 0x12ed785 0x332 c407aa1 0x332c0e8bcd ----- BEGIN BACKTRACE ----- {"backtrace":[\{"b":"400000","o":"F42B42","s":"_ZN5mongo15printStackTraceERSo"},\{"b":"400000","o":"EDE108","s":"_ZN5mongo10logContextEPKc"},\{"b":"400000","o":"EC6448","s":"_ZN5mongo11msgassertedEiPKc"} ,\{"b":"400000","o":"EC64FC"},\{"b":"400000","o":"5BCCDB","s":"_ZNK5mongo11BSONElement3chkEi"},{"b":"400000","o":"779737","s":"_ZN5mongo21updateUserCredentialsEPNS_16OperationContextERKNS_10StringDataER KNS_7BSONObjES7_"},{"b":"400000","o":"77D6C8","s":"_ZNSt17_Function_handlerIFvRKN5mongo7BSONObjEESt5_BindIFPFvPNS0_16OperationContextERKNS0_10StringDataES3_S3_ES7_PKcSt12_PlaceholderILi1EES1_EEE9_M_in vokeERKSt9_Any_dataS3_"},\{"b":"400000","o":"60671B","s":"_ZNSt17_Function_handlerIFvRN5mongo27DBClientCursorBatchIteratorEENS0_20DBClientFunConvertorEE9_M_invokeERKSt9_Any_dataS2_"},{"b":"400000","o":( "5FA7BA","s":"_ZN5mongo12DBClientBase5queryESt8functionIFvRNS_27DBClientCursorBatchIteratorEEERKSsNS_5QueryEPKNS_7BSONObjEi"},{"b":"400000","o":"5FA602","s":"_ZN5mongo12DBClientBase5queryESt8functionI FvRKNS_7BSONObjEEERKSsNS_5QueryEPS3_i"},\{"b":"400000","o":"77681A"},\{"b":"400000","o":"77A1DD","s":"_ZN5mongo17updateCredentialsEPNS_16OperationContextERKNS_7BSONObjE"},{"b":"400000","o":"77A67F","s": "_ZN5mongo21upgradeAuthSchemaStepEPNS_16OperationContextEPNS_20AuthorizationManagerERKNS_7BSONObjEPb"},{"b":"400000","o":"77A7A2","s":"_ZN5mongo17upgradeAuthSchemaEPNS_16OperationContextEPNS_20Authori zationManagerEiRKNS_7BSONObjE"},\{"b":"400000","o":"78083B","s":"_ZN5mongo20CmdAuthSchemaUpgrade3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderE"},{"b":"400000","o":"7BACE3","s":" _ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE"},{"b":"400000","o":"7BBB74","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_ 3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},\{"b":"400000","o":"711FF0","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"4 00000","o":"8D1D45","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"59EBCC","s":"_ZN5mongo16MyMessageHandler7processERNS_ 7MessageEPNS_21AbstractMessagingPortE"},\{"b":"400000","o":"EED785","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},\{"b":"332C400000","o":"7AA1"},\{"b":"332C000000","o":"E8BCD","s":"clone"}]," processInfo":{ "mongodbVersion" : "3.2.10", "gitVersion" : "79d9b3ab5ce20f51c272b4411202710a082d0317", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "2.6.32-696.3.2.el6.x86_64", "version" : "#1 SMP Tue Jun 20 01:26:55 UTC 2017", "machine" : "x86_64" }, "somap" : [ \{ "elfType" : 2, "b" : "400000", "buildId" : "52E5CD214869E2C4253E66CA8F1FB295ACD34936" }, { "b" : "7FFE9F282000 ", "elfType" : 3, "buildId" : "9AEB3ED69C98EE1CFF1DB57C31AD439C40DF044A" }, \{ "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "BECFB85A8BC084042D5BF2BA9E66325CE798B659" }, { "path" : "/ usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "CBDA444A7109874C5350AE9CEEF3F82F749B347F" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "FDF3A36FFFE08375456D59DA959EAB2FC30B618 6" }, \{ "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "1F7E85410384392BC51FA7324961719A10125F31" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "8A852AC42F0B64F0F30C760EBBCFA3 FE4A228F12" }, \{ "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "BC7550A8A7C2D706FE4E489058BADC963465DBB7" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "85104ECFE42C 606B31C2D0D0D2E5DACD3286A341" }, \{ "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "814F2290D172521A3FD8581389E3E78A4A182379" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "build Id" : "1CC2165E019D43F71FDE0A47AF9F4C8EB5E51963" }, \{ "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9A737F8BF10FC99C37CC404D3FC188F6E11FEDD9" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "8D3D6E28DF6EB3752642A7031AAC17D39EA4265D" }, \{ "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "57F77704A7F1F4E3689D028D3F9ADD4E77486EC9" }, { "path" : "/lib6 4/libk5crypto.so.3", "elfType" : 3, "buildId" : "CC89B4C8CDCCD32BA610BC72784DC3B7E9BD9E19" }, \{ "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E0C522C589F775C324330BE09CE67DC83950A213" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431 D3F06D82014AFF3251" }, \{ "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F0BE1166EDCFFB2422B940D601A1BBD89352D80F" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "B45 76BE308DDCF7BC31F7304E4734C3D846D0236" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1342b42] mongod(_ZN5mongo10logContextEPKc+0x138) [0x12de108] mongod(_ZN5mongo11msgassertedEiPKc+0x88) [0x12c6448] mongod(+0xEC64FC) [0x12c64fc] mongod(_ZNK5mongo11BSONElement3chkEi+0x1CB) [0x9bccdb] mongod(_ZN5mongo21updateUserCredentialsEPNS_16OperationContextERKNS_10StringDataERKNS_7BSONObjES7_+0x67) [0xb79737] mongod(_ZNSt17_Function_handlerIFvRKN5mongo7BSONObjEESt5_BindIFPFvPNS0_16OperationContextERKNS0_10StringDataES3_S3_ES7_PKcSt12_PlaceholderILi1EES1_EEE9_M_invokeERKSt9_Any_dataS3_+0x48) [0xb7d6c8] mongod(_ZNSt17_Function_handlerIFvRN5mongo27DBClientCursorBatchIteratorEENS0_20DBClientFunConvertorEE9_M_invokeERKSt9_Any_dataS2_+0x4B) [0xa0671b] mongod(_ZN5mongo12DBClientBase5queryESt8functionIFvRNS_27DBClientCursorBatchIteratorEEERKSsNS_5QueryEPKNS_7BSONObjEi+0x9A) [0x9fa7ba] mongod(_ZN5mongo12DBClientBase5queryESt8functionIFvRKNS_7BSONObjEEERKSsNS_5QueryEPS3_i+0x1D2) [0x9fa602] mongod(+0x77681A) [0xb7681a] mongod(_ZN5mongo17updateCredentialsEPNS_16OperationContextERKNS_7BSONObjE+0x17D) [0xb7a1dd] mongod(_ZN5mongo21upgradeAuthSchemaStepEPNS_16OperationContextEPNS_20AuthorizationManagerERKNS_7BSONObjEPb+0x18F) [0xb7a67f] mongod(_ZN5mongo17upgradeAuthSchemaEPNS_16OperationContextEPNS_20AuthorizationManagerEiRKNS_7BSONObjE+0x72) [0xb7a7a2] mongod(_ZN5mongo20CmdAuthSchemaUpgrade3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderE+0xEB) [0xb8083b] mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x473) [0xbbace3] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0x404) [0xbbbb74] mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x1F0) [0xb11ff0] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB45) [0xcd1d45] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE+0xEC) [0x99ebcc] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x325) [0x12ed785] libpthread.so.0(+0x7AA1) [0x332c407aa1] libc.so.6(clone+0x6D) [0x332c0e8bcd] ----- END BACKTRACE -----