diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 5f560b6fc5..64a54953b3 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -27,7 +27,7 @@ * it in the license file. */ -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kWiredTiger #include "mongo/platform/basic.h" @@ -525,12 +525,163 @@ size_t WiredTigerUtil::getCacheSizeMB(double requestedCacheSizeGB) { return static_cast(cacheSizeMB); } +logv2::LogSeverity getWTLOGV2SeverityLevel(const BSONObj& obj) { + const std::string field = "verbose_level_id"; + + if (!obj.hasField(field)) { + throw std::logic_error("Missing field: " + field); + } + + BSONElement verbose_level_id_ele = obj[field]; + if (!verbose_level_id_ele.isNumber()) { + throw std::logic_error(field + " is NaN"); + } + + std::cout << field << " is " << verbose_level_id_ele.Int() << std::endl; + + auto level = logv2::LogSeverity::Log(); + switch (verbose_level_id_ele.Int()) { + case WT_VERBOSE_ERROR: + level = logv2::LogSeverity::Error(); + break; + case WT_VERBOSE_WARNING: + level = logv2::LogSeverity::Warning(); + break; + case WT_VERBOSE_NOTICE: + case WT_VERBOSE_INFO: + level = logv2::LogSeverity::Info(); + break; + case WT_VERBOSE_DEBUG: + level = logv2::LogSeverity::Debug(1); + break; + } + return level; +} + +logv2::LogComponent getWTLOGV2Component(const BSONObj& obj) { + const std::string field = "category_id"; + + if (!obj.hasField(field)) { + throw std::logic_error("Missing field: " + field); + } + + BSONElement category_id_ele = obj[field]; + if (!category_id_ele.isNumber()) { + throw std::logic_error(field + " is NaN"); + } + + std::cout << field << " is " << category_id_ele.Int() << std::endl; + + auto component = logv2::LogComponent::kWiredTiger; + switch (category_id_ele.Int()) { + case WT_VERB_BACKUP: + component = logv2::LogComponent::kWiredTigerBackup; + break; + case WT_VERB_CHECKPOINT: + case WT_VERB_CHECKPOINT_CLEANUP: + // case WT_VERB_CHECKPOINT_PROGRESS: + component = logv2::LogComponent::kWiredTigerCheckpoint; + break; + case WT_VERB_COMPACT: + // case WT_VERB_COMPACT_PROGRESS: + component = logv2::LogComponent::kWiredTigerCompact; + break; + case WT_VERB_EVICT: + case WT_VERB_EVICTSERVER: + case WT_VERB_EVICT_STUCK: + component = logv2::LogComponent::kWiredTigerEviction; + break; + case WT_VERB_HS: + case WT_VERB_HS_ACTIVITY: + component = logv2::LogComponent::kWiredTigerHS; + break; + case WT_VERB_RECOVERY: + // case WT_VERB_RECOVERY_PROGRESS: + component = logv2::LogComponent::kWiredTigerRecovery; + break; + case WT_VERB_RTS: + component = logv2::LogComponent::kWiredTigerRTS; + break; + case WT_VERB_SALVAGE: + case WT_VERB_SALVAGE_PROGRESS: + component = logv2::LogComponent::kWiredTigerSalvage; + break; + case WT_VERB_TIERED: + component = logv2::LogComponent::kWiredTigerTiered; + break; + case WT_VERB_TIMESTAMP: + component = logv2::LogComponent::kWiredTigerTimestamp; + break; + case WT_VERB_TRANSACTION: + component = logv2::LogComponent::kWiredTigerTransaction; + break; + case WT_VERB_VERIFY: + component = logv2::LogComponent::kWiredTigerVerify; + break; + case WT_VERB_WRITE: + component = logv2::LogComponent::kWiredTigerWriteLog; + break; + } + + return component; +} + +void buildWTMessage(const BSONObj& obj, logv2::DynamicAttributes& attrs) { + + if (!obj.hasField("msg")) { + throw std::logic_error("Missing field: msg"); + } + std::cout << "msg is " << obj["msg"].str() << std::endl; + attrs.add("msg", redact(obj["msg"].valuestrsafe())); + + if (obj.hasField("session_dhandle_name")) { + attrs.add("session_dhandle_name", (obj["session_dhandle_name"].valuestrsafe())); + } + + if (obj.hasField("session_err_prefix")) { + attrs.add("session_err_prefix", StringData(obj["session_err_prefix"].valuestrsafe())); + } + + if (obj.hasField("session_name")) { + attrs.add("session_name", StringData(obj["session_name"].valuestrsafe())); + } +} + namespace { + +void logJSONErrorMessage(int id, int errorCode, const char* message) { + try { + /* Parse the WT message. */ + BSONObj obj = fromjson(message); + + /* Build the message. */ + auto options = logv2::LogOptions{getWTLOGV2Component(obj)}; + auto level = getWTLOGV2SeverityLevel(obj); + logv2::DynamicAttributes attrs; + buildWTMessage(obj, attrs); + + /* Add the value associated with the error_str field. */ + if (obj.hasField("error_str")) { + attrs.add("error_str", (obj["error_str"].valuestrsafe())); + } + + LOGV2_IMPL(id, level, options, "WiredTiger error", attrs); + } catch (...) { + /* Fall back to default behaviour. */ + LOGV2_ERROR(id, + "WiredTiger error ({errorCode}) {message}", + "WiredTiger error", + "error"_attr = errorCode, + "message"_attr = message); + } +} + int mdb_handle_error_with_startup_suppression(WT_EVENT_HANDLER* handler, WT_SESSION* session, int errorCode, const char* message) { WiredTigerEventHandler* wtHandler = reinterpret_cast(handler); + std::cout << "mdb_handle_error_with_startup_suppression message is " << message << std::endl; try { StringData sd(message); @@ -551,11 +702,18 @@ int mdb_handle_error_with_startup_suppression(WT_EVENT_HANDLER* handler, return 0; } } - LOGV2_ERROR(22435, - "WiredTiger error ({error}) {message}", - "WiredTiger error", - "error"_attr = errorCode, - "message"_attr = message); + + WiredTigerEventHandler* wtHandler = reinterpret_cast(handler); + + if (!wtHandler->isJSONOutput()) { + LOGV2_ERROR(22435, + "WiredTiger error ({error}) {message}", + "WiredTiger error", + "error"_attr = errorCode, + "message"_attr = message); + } else { + logJSONErrorMessage(22435, errorCode, message); + } // Don't abort on WT_PANIC when repairing, as the error will be handled at a higher layer. if (storageGlobalParams.repair) { @@ -572,12 +730,18 @@ int mdb_handle_error(WT_EVENT_HANDLER* handler, WT_SESSION* session, int errorCode, const char* message) { + std::cout << "mdb_handle_error message is " << message << std::endl; + WiredTigerEventHandler* wtHandler = reinterpret_cast(handler); try { - LOGV2_ERROR(22436, - "WiredTiger error ({errorCode}) {message}", - "WiredTiger error", - "error"_attr = errorCode, - "message"_attr = redact(message)); + if (!wtHandler->isJSONOutput()) { + LOGV2_ERROR(22436, + "WiredTiger error ({errorCode}) {message}", + "WiredTiger error", + "error"_attr = errorCode, + "message"_attr = redact(message)); + } else { + logJSONErrorMessage(22436, errorCode, message); + } // Don't abort on WT_PANIC when repairing, as the error will be handled at a higher layer. if (storageGlobalParams.repair) { @@ -591,8 +755,30 @@ int mdb_handle_error(WT_EVENT_HANDLER* handler, } int mdb_handle_message(WT_EVENT_HANDLER* handler, WT_SESSION* session, const char* message) { + + WiredTigerEventHandler* wtHandler = reinterpret_cast(handler); + std::cout << "mdb_handle_message message is " << message << std::endl; try { - LOGV2(22430, "WiredTiger message", "message"_attr = redact(message)); + if (!wtHandler->isJSONOutput()) { + LOGV2(22430, "WiredTiger message", "message"_attr = redact(message)); + } else { + + try { + /* Parse the WT message. */ + BSONObj obj = fromjson(message); + + /* Build the message. */ + auto options = logv2::LogOptions{getWTLOGV2Component(obj)}; + auto level = getWTLOGV2SeverityLevel(obj); + logv2::DynamicAttributes attrs; + buildWTMessage(obj, attrs); + + LOGV2_IMPL(22430, level, options, "WiredTiger message", attrs); + } catch (...) { + /* Fall back to default behaviour. */ + LOGV2(22430, "WiredTiger message", "message"_attr = redact(message)); + } + } } catch (...) { std::terminate(); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 3b7357de73..94bfad85ca 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -139,9 +139,20 @@ public: _wtIncompatible = true; } + bool isJSONOutput() { + return _isJSONOutput; + } + + void setJSONOutput(bool value) { + _isJSONOutput = value; + } + private: bool _startupSuccessful = false; bool _wtIncompatible = false; + // TODO - Define a new arg that can be parsed from the command line options + // bool _isJSONOutput = false; + bool _isJSONOutput = true; }; class WiredTigerUtil { diff --git a/src/mongo/logv2/log_component.h b/src/mongo/logv2/log_component.h index 22d18d0540..a6ee65c16a 100644 --- a/src/mongo/logv2/log_component.h +++ b/src/mongo/logv2/log_component.h @@ -74,6 +74,20 @@ namespace mongo::logv2 { X(kStorage, , "storage" , "STORAGE" , kDefault) \ X(kStorageRecovery, , "recovery" , "RECOVERY", kStorage) \ X(kJournal, , "journal" , "JOURNAL" , kStorage) \ + X(kWiredTiger, , "wiredtiger" , "WT" , kStorage) \ + X(kWiredTigerBackup, , "wiredtigerBackup" , "WTBACKUP", kWiredTiger) \ + X(kWiredTigerCheckpoint, , "wiredtigerCheckpoint" , "WTCHKPT" , kWiredTiger) \ + X(kWiredTigerCompact, , "wiredtigerCompact" , "WTCMPCT" , kWiredTiger) \ + X(kWiredTigerEviction, , "wiredtigerEviction" , "WTEVICT" , kWiredTiger) \ + X(kWiredTigerHS, , "wiredtigerHS" , "WTHS" , kWiredTiger) \ + X(kWiredTigerRecovery, , "wiredtigerRecovery" , "WTRECOV" , kWiredTiger) \ + X(kWiredTigerRTS, , "wiredtigerRTS" , "WTRTS" , kWiredTiger) \ + X(kWiredTigerSalvage, , "wiredtigerSalvage" , "WTSLVG" , kWiredTiger) \ + X(kWiredTigerTiered, , "wiredtigerTiered" , "WTTIER" , kWiredTiger) \ + X(kWiredTigerTimestamp, , "wiredtigerTimestamp" , "WTTS" , kWiredTiger) \ + X(kWiredTigerTransaction, , "wiredtigerTransaction" , "WTTXN" , kWiredTiger) \ + X(kWiredTigerVerify, , "wiredtigerVerify" , "WTVRFY" , kWiredTiger) \ + X(kWiredTigerWriteLog, , "wiredTigerWriteLog" , "WTWRTLOG", kWiredTiger) \ X(kWrite, , "write" , "WRITE" , kDefault) \ X(kFTDC, , "ftdc" , "FTDC" , kDefault) \ X(kASIO, , "asio" , "ASIO" , kNetwork) \ diff --git a/src/third_party/wiredtiger/src/include/verbose.h b/src/third_party/wiredtiger/src/include/verbose.h index b170108440..167b374e2f 100644 --- a/src/third_party/wiredtiger/src/include/verbose.h +++ b/src/third_party/wiredtiger/src/include/verbose.h @@ -6,84 +6,59 @@ * See the file LICENSE for redistribution information. */ -#define WT_GEN_VERBOSE_ENUM(value) value, -#define WT_GEN_VERBOSE_ENUM_STR(value_str) #value_str, - -/* Disable clang format for the following macro to keep the categories formatted on a new line. */ -/* clang-format off */ -#define WT_GEN_VERBOSE_CATEGORIES(gen) \ - gen(WT_VERB_API) \ - gen(WT_VERB_BACKUP) \ - gen(WT_VERB_BLKCACHE) \ - gen(WT_VERB_BLOCK) \ - gen(WT_VERB_CHECKPOINT) \ - gen(WT_VERB_CHECKPOINT_CLEANUP) \ - gen(WT_VERB_CHECKPOINT_PROGRESS) \ - gen(WT_VERB_COMPACT) \ - gen(WT_VERB_COMPACT_PROGRESS) \ - gen(WT_VERB_DEFAULT) \ - gen(WT_VERB_ERROR_RETURNS) \ - gen(WT_VERB_EVICT) \ - gen(WT_VERB_EVICTSERVER) \ - gen(WT_VERB_EVICT_STUCK) \ - gen(WT_VERB_EXTENSION) \ - gen(WT_VERB_FILEOPS) \ - gen(WT_VERB_GENERATION) \ - gen(WT_VERB_HANDLEOPS) \ - gen(WT_VERB_HS) \ - gen(WT_VERB_HS_ACTIVITY) \ - gen(WT_VERB_LOG) \ - gen(WT_VERB_LSM) \ - gen(WT_VERB_LSM_MANAGER) \ - gen(WT_VERB_METADATA) \ - gen(WT_VERB_MUTEX) \ - gen(WT_VERB_OUT_OF_ORDER) \ - gen(WT_VERB_OVERFLOW) \ - gen(WT_VERB_READ) \ - gen(WT_VERB_RECONCILE) \ - gen(WT_VERB_RECOVERY) \ - gen(WT_VERB_RECOVERY_PROGRESS) \ - gen(WT_VERB_RTS) \ - gen(WT_VERB_SALVAGE) \ - gen(WT_VERB_SALVAGE_PROGRESS) \ - gen(WT_VERB_SHARED_CACHE) \ - gen(WT_VERB_SPLIT) \ - gen(WT_VERB_TEMPORARY) \ - gen(WT_VERB_THREAD_GROUP) \ - gen(WT_VERB_TIERED) \ - gen(WT_VERB_TIMESTAMP) \ - gen(WT_VERB_TRANSACTION) \ - gen(WT_VERB_UPGRADE_PROGRESS) \ - gen(WT_VERB_VERIFY) \ - gen(WT_VERB_VERIFY_PROGRESS) \ - gen(WT_VERB_VERSION) \ - gen(WT_VERB_WRITE) -/* clang-format on */ - -/* Permitted verbose event categories that can be used when defining a verbose message. */ -typedef enum { - WT_GEN_VERBOSE_CATEGORIES(WT_GEN_VERBOSE_ENUM) - /* This entry needs to be the last in order to track the number of category items. */ - WT_VERB_NUM_CATEGORIES, -} WT_VERBOSE_CATEGORY; - -/* Convert a category to its string representation. */ +// clang-format off #define WT_VERBOSE_CATEGORY_STR(category) \ - (const char *[]){WT_GEN_VERBOSE_CATEGORIES(WT_GEN_VERBOSE_ENUM_STR)}[category] - -/* - * Permitted verbosity levels; to be used when defining verbose messages. The levels define a range - * of severity categories, with WT_VERBOSE_ERROR being the lowest, most critical level (used by - * messages on critical error paths) and WT_VERBOSE_DEBUG being the highest verbosity/informational - * level (mostly adopted for debugging). - */ -typedef enum { - WT_VERBOSE_ERROR = -3, - WT_VERBOSE_WARNING, - WT_VERBOSE_NOTICE, - WT_VERBOSE_INFO, - WT_VERBOSE_DEBUG -} WT_VERBOSE_LEVEL; + (const char *[]){ \ + /* AUTOMATIC VERBOSE ENUM STRING GENERATION START */ \ + "WT_VERB_API", \ + "WT_VERB_BACKUP", \ + "WT_VERB_BLKCACHE", \ + "WT_VERB_BLOCK", \ + "WT_VERB_CHECKPOINT", \ + "WT_VERB_CHECKPOINT_CLEANUP", \ + "WT_VERB_CHECKPOINT_PROGRESS", \ + "WT_VERB_COMPACT", \ + "WT_VERB_COMPACT_PROGRESS", \ + "WT_VERB_DEFAULT", \ + "WT_VERB_ERROR_RETURNS", \ + "WT_VERB_EVICT", \ + "WT_VERB_EVICTSERVER", \ + "WT_VERB_EVICT_STUCK", \ + "WT_VERB_EXTENSION", \ + "WT_VERB_FILEOPS", \ + "WT_VERB_GENERATION", \ + "WT_VERB_HANDLEOPS", \ + "WT_VERB_HS", \ + "WT_VERB_HS_ACTIVITY", \ + "WT_VERB_LOG", \ + "WT_VERB_LSM", \ + "WT_VERB_LSM_MANAGER", \ + "WT_VERB_MUTEX", \ + "WT_VERB_METADATA", \ + "WT_VERB_OUT_OF_ORDER", \ + "WT_VERB_OVERFLOW", \ + "WT_VERB_READ", \ + "WT_VERB_RECONCILE", \ + "WT_VERB_RECOVERY", \ + "WT_VERB_RECOVERY_PROGRESS", \ + "WT_VERB_RTS", \ + "WT_VERB_SALVAGE", \ + "WT_VERB_SALVAGE_PROGRESS", \ + "WT_VERB_SHARED_CACHE", \ + "WT_VERB_SPLIT", \ + "WT_VERB_TEMPORARY", \ + "WT_VERB_THREAD_GROUP", \ + "WT_VERB_TIERED", \ + "WT_VERB_TIMESTAMP", \ + "WT_VERB_TRANSACTION", \ + "WT_VERB_UPGRADE_PROGRESS", \ + "WT_VERB_VERIFY", \ + "WT_VERB_VERIFY_PROGRESS", \ + "WT_VERB_VERSION", \ + "WT_VERB_WRITE", \ + /* AUTOMATIC VERBOSE ENUM STRING GENERATION STOP */ \ + }[category] +// clang-format on /* Convert a verbose level to its string representation. */ #define WT_VERBOSE_LEVEL_STR(level, level_str) \ diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 5f4efde2e6..c44241f16d 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -7075,6 +7075,103 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); */ /*! @} */ +/******************************************* + * Verbose categories + *******************************************/ +/*! + * @addtogroup wt + * @{ + */ +/*! + * @name Verbose categories + * @anchor verbose_categories + * @{ + */ +/*! + * WiredTiger verbose event categories. + */ +typedef enum { +/* VERBOSE ENUM START */ + WT_VERB_API, /*!< API messages. */ + WT_VERB_BACKUP, /*!< Backup messages. */ + WT_VERB_BLKCACHE, /*!< Block cache messages. */ + WT_VERB_BLOCK, /*!< Block manager messages. */ + WT_VERB_CHECKPOINT, /*!< Checkpoint messages. */ + WT_VERB_CHECKPOINT_CLEANUP, /*!< Checkpoint cleanup messages. */ + WT_VERB_CHECKPOINT_PROGRESS, /*!< Checkpoint progress messages. */ + WT_VERB_COMPACT, /*!< Compact messages. */ + WT_VERB_COMPACT_PROGRESS, /*!< Compact progress messages. */ + WT_VERB_DEFAULT, /*!< Default messages. */ + WT_VERB_ERROR_RETURNS, /*!< Error returns messages. */ + WT_VERB_EVICT, /*!< Eviction messages. */ + WT_VERB_EVICTSERVER, /*!< Eviction server messages. */ + WT_VERB_EVICT_STUCK, /*!< Eviction stuck messages. */ + WT_VERB_EXTENSION, /*!< Extension messages. */ + WT_VERB_FILEOPS, /*!< File operation messages. */ + WT_VERB_GENERATION, /*!< Generation messages. */ + WT_VERB_HANDLEOPS, /*!< Handle operation messages. */ + WT_VERB_HS, /*!< History store messages. */ + WT_VERB_HS_ACTIVITY, /*!< History store activity messages. */ + WT_VERB_LOG, /*!< Log messages. */ + WT_VERB_LSM, /*!< LSM messages. */ + WT_VERB_LSM_MANAGER, /*!< LSM manager messages. */ + WT_VERB_MUTEX, /*!< API messages. */ + WT_VERB_METADATA, /*!< Metadata messages. */ + WT_VERB_OUT_OF_ORDER, /*!< Out of order messages. */ + WT_VERB_OVERFLOW, /*!< Overflow messages. */ + WT_VERB_READ, /*!< Read messages. */ + WT_VERB_RECONCILE, /*!< Reconcile messages. */ + WT_VERB_RECOVERY, /*!< Recovery messages. */ + WT_VERB_RECOVERY_PROGRESS, /*!< Recovery progress messages. */ + WT_VERB_RTS, /*!< RTS messages. */ + WT_VERB_SALVAGE, /*!< Salvage messages. */ + WT_VERB_SALVAGE_PROGRESS, /*!< Salvage progress messages. */ + WT_VERB_SHARED_CACHE, /*!< Shared cache messages. */ + WT_VERB_SPLIT, /*!< Split messages. */ + WT_VERB_TEMPORARY, /*!< Temporary messages. */ + WT_VERB_THREAD_GROUP, /*!< Thread group messages. */ + WT_VERB_TIERED, /*!< Tiered messages. */ + WT_VERB_TIMESTAMP, /*!< Timestamp messages. */ + WT_VERB_TRANSACTION, /*!< Transaction messages. */ + WT_VERB_UPGRADE_PROGRESS, /*!< Update progress messages. */ + WT_VERB_VERIFY, /*!< Verify messages. */ + WT_VERB_VERIFY_PROGRESS, /*!< Verify progress messages. */ + WT_VERB_VERSION, /*!< Version progress messages. */ + WT_VERB_WRITE, /*!< Write progress messages. */ +/* VERBOSE ENUM STOP */ + WT_VERB_NUM_CATEGORIES +} WT_VERBOSE_CATEGORY; +/*! @} */ + +/******************************************* + * Verbose levels + *******************************************/ +/*! + * @name Verbose levels + * @anchor verbose_levels + * @{ + */ +/*! + * WiredTiger verbosity levels. The levels define a range of severity categories, with + * \c WT_VERBOSE_ERROR being the lowest, most critical level (used by messages on critical error + * paths) and \c WT_VERBOSE_DEBUG being the highest verbosity/informational level (mostly adopted + * for debugging). + */ +typedef enum { + WT_VERBOSE_ERROR = -3, /*!< Error conditions triggered in WiredTiger. */ + WT_VERBOSE_WARNING, /*!< Warning conditions potentially signaling non-imminent errors and + behaviors. */ + WT_VERBOSE_NOTICE, /*!< Messages for significant events in WiredTiger, usually worth + noting. */ + WT_VERBOSE_INFO, /*!< Informational style messages. */ + WT_VERBOSE_DEBUG /*!< Low severity messages, useful for debugging purposes. */ +} WT_VERBOSE_LEVEL; +/*! @} */ +/* + * Verbose section: END + */ +/*! @} */ + #undef __F #if defined(__cplusplus)