diff --git a/src/include/rollback_to_stable.h b/src/include/rollback_to_stable.h index fb470909c..d0216a5d6 100644 --- a/src/include/rollback_to_stable.h +++ b/src/include/rollback_to_stable.h @@ -19,6 +19,7 @@ #define WT_RTS_VERB_TAG_HS_TREE_SKIP "[HS_TREE_SKIP] " #define WT_RTS_VERB_TAG_HS_TRUNCATED "[HS_TRUNCATED] " #define WT_RTS_VERB_TAG_HS_UPDATE_ABORT "[HS_UPDATE_ABORT] " +#define WT_RTS_VERB_TAG_HS_UPDATE_REMOVE "[HS_UPDATE_REMOVE] " #define WT_RTS_VERB_TAG_HS_UPDATE_RESTORED "[HS_UPDATE_RESTORED] " #define WT_RTS_VERB_TAG_HS_UPDATE_VALID "[HS_UPDATE_VALID] " #define WT_RTS_VERB_TAG_INIT "[INIT] " @@ -39,6 +40,7 @@ #define WT_RTS_VERB_TAG_TREE_LOGGING "[TREE_LOGGING] " #define WT_RTS_VERB_TAG_TREE_SKIP "[TREE_SKIP] " #define WT_RTS_VERB_TAG_UPDATE_ABORT "[UPDATE_ABORT] " +#define WT_RTS_VERB_TAG_UPDATE_VALID "[UPDATE_VALID] " #define WT_CHECK_RECOVERY_FLAG_TXNID(session, txnid) \ (F_ISSET(S2C(session), WT_CONN_RECOVERING) && S2C(session)->recovery_ckpt_snap_min != 0 && \ diff --git a/src/rollback_to_stable/rts_btree.c b/src/rollback_to_stable/rts_btree.c index 3e3257bcb..08ea059bf 100644 --- a/src/rollback_to_stable/rts_btree.c +++ b/src/rollback_to_stable/rts_btree.c @@ -17,6 +17,7 @@ static int __rts_btree_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *first_upd, wt_timestamp_t rollback_timestamp, bool *stable_update_found) { + WT_DECL_ITEM(key_string); WT_UPDATE *stable_upd, *tombstone, *upd; char ts_string[2][WT_TS_INT_STRING_SIZE]; bool dryrun; @@ -24,6 +25,11 @@ __rts_btree_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *firs dryrun = S2C(session)->rts->dryrun; + WT_RET(__wt_scr_alloc(session, 0, &key_string)); + __wt_verbose_level_multi(session, WT_VERB_RECOVERY_RTS(session), WT_VERBOSE_DEBUG_2, + WT_RTS_VERB_TAG_ONDISK_KEY_ROLLBACK "verifying update list key=%s", + __wt_key_string(session, key->data, key->size, S2BT(session)->key_format, key_string)); + stable_upd = tombstone = NULL; txn_id_visible = false; if (stable_update_found != NULL) @@ -64,6 +70,12 @@ __rts_btree_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *firs } else { /* Valid update is found. */ stable_upd = upd; + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), + WT_RTS_VERB_TAG_UPDATE_VALID + "stable update found with txnid=%" PRIu64 + ", flags=0x%x, durable_timestamp=%s, commit_timestamp=%s", + upd->txnid, upd->flags, __wt_timestamp_to_string(upd->durable_ts, ts_string[0]), + __wt_timestamp_to_string(upd->start_ts, ts_string[1])); break; } } @@ -113,6 +125,7 @@ __rts_btree_abort_update(WT_SESSION_IMPL *session, WT_ITEM *key, WT_UPDATE *firs *stable_update_found = true; } + __wt_scr_free(session, &key_string); return (0); } diff --git a/src/rollback_to_stable/rts_history.c b/src/rollback_to_stable/rts_history.c index b788a2215..a0a015aec 100644 --- a/src/rollback_to_stable/rts_history.c +++ b/src/rollback_to_stable/rts_history.c @@ -20,6 +20,7 @@ __wt_rts_history_delete_hs(WT_SESSION_IMPL *session, WT_ITEM *key, wt_timestamp_ WT_DECL_ITEM(hs_key); WT_DECL_RET; WT_TIME_WINDOW *hs_tw; + char time_string[WT_TIME_STRING_SIZE]; bool dryrun; dryrun = S2C(session)->rts->dryrun; @@ -54,6 +55,11 @@ __wt_rts_history_delete_hs(WT_SESSION_IMPL *session, WT_ITEM *key, wt_timestamp_ if (hs_tw->stop_ts <= ts) break; + __wt_verbose_multi(session, WT_VERB_RECOVERY_RTS(session), + WT_RTS_VERB_TAG_HS_UPDATE_REMOVE + "rollback to stable aborting hs update with time window=%s", + __wt_time_window_to_string(hs_tw, time_string)); + if (!dryrun) WT_ERR(hs_cursor->remove(hs_cursor)); WT_RTS_STAT_CONN_DATA_INCR(session, txn_rts_hs_removed); diff --git a/tools/rts_verifier/checker.py b/tools/rts_verifier/checker.py index 2e3b07177..a3e920e6d 100644 --- a/tools/rts_verifier/checker.py +++ b/tools/rts_verifier/checker.py @@ -90,6 +90,10 @@ class Checker: if not operation.stable_lt_durable and not operation.stable >= operation.durable: raise Exception(f"incorrect timestamp comparison: thought {operation.stable} >= {operation.durable}, but it isn't") + def __apply_check_update_valid(self, operation): + # TODO expand this out + pass + def __apply_check_page_abort_check(self, operation): # TODO expand this out # if operation.file != self.current_tree.file: @@ -160,7 +164,7 @@ class Checker: # TODO expand this out pass - def __apply_check_hs_update_valid(self, operation): + def __apply_check_hs_update_remove(self, operation): # TODO expand this out pass @@ -168,6 +172,10 @@ class Checker: # TODO expand this out pass + def __apply_check_hs_update_valid(self, operation): + # TODO expand this out + pass + def __apply_check_key_removed(self, operation): # TODO expand this out pass diff --git a/tools/rts_verifier/operation.py b/tools/rts_verifier/operation.py index 440e304b5..d8a3c40a6 100644 --- a/tools/rts_verifier/operation.py +++ b/tools/rts_verifier/operation.py @@ -11,33 +11,35 @@ class OpType(Enum): TREE_LOGGING = 2 PAGE_ROLLBACK = 3 UPDATE_ABORT = 4 - PAGE_ABORT_CHECK = 5 - KEY_CLEAR_REMOVE = 6 - ONDISK_KV_REMOVE = 7 - SHUTDOWN_INIT = 8 - TREE_SKIP = 9 - SKIP_DEL_NULL = 10 - ONDISK_ABORT_TW = 11 - ONDISK_KEY_ROLLBACK = 12 - HS_UPDATE_ABORT = 13 - HS_UPDATE_VALID = 14 - HS_UPDATE_RESTORED = 15 - KEY_REMOVED = 16 - STABLE_PG_WALK_SKIP = 17 - SKIP_UNMODIFIED = 18 - HS_GT_ONDISK = 19 - RECOVERY_RTS = 20 - HS_STOP_OBSOLETE = 21 - RECOVER_CKPT = 22 - HS_TREE_ROLLBACK = 23 - HS_TREE_SKIP = 24 - HS_ABORT_STOP = 25 - HS_RESTORE_TOMBSTONE = 26 - FILE_SKIP = 27 - SKIP_DAMAGE = 28 - HS_TRUNCATED = 29 - SHUTDOWN_RTS = 30 - END = 31 + UPDATE_VALID = 5 + PAGE_ABORT_CHECK = 6 + KEY_CLEAR_REMOVE = 7 + ONDISK_KV_REMOVE = 8 + SHUTDOWN_INIT = 9 + TREE_SKIP = 10 + SKIP_DEL_NULL = 11 + ONDISK_ABORT_TW = 12 + ONDISK_KEY_ROLLBACK = 13 + HS_UPDATE_ABORT = 14 + HS_UPDATE_REMOVE = 15 + HS_UPDATE_RESTORED = 16 + HS_UPDATE_VALID = 17 + KEY_REMOVED = 18 + STABLE_PG_WALK_SKIP = 19 + SKIP_UNMODIFIED = 20 + HS_GT_ONDISK = 21 + RECOVERY_RTS = 22 + HS_STOP_OBSOLETE = 23 + RECOVER_CKPT = 24 + HS_TREE_ROLLBACK = 25 + HS_TREE_SKIP = 26 + HS_ABORT_STOP = 27 + HS_RESTORE_TOMBSTONE = 28 + FILE_SKIP = 29 + SKIP_DAMAGE = 30 + HS_TRUNCATED = 31 + SHUTDOWN_RTS = 32 + END = 33 class Operation: def __init__(self, line): @@ -167,6 +169,13 @@ class Operation: matches = re.search('prepare_state=(\w+)', line) self.prepare_state = PrepareState[matches.group(1)] + def __init_update_valid(self, line): + self.type = OpType.UPDATE_VALID + self.file = self.__extract_file(line) + + matches = re.search('txnid=(\d+)', line) + self.txnid = int(matches.group(1)) + def __init_page_abort_check(self, line): self.type = OpType.PAGE_ABORT_CHECK self.file = self.__extract_file(line) @@ -323,8 +332,30 @@ class Operation: self.stable = self.__extract_simple_timestamp('stable_timestamp', line) + def __init_hs_update_remove(self, line): + self.type = OpType.HS_UPDATE_REMOVE + self.file = self.__extract_file(line) + + matches = re.search('time window=start: \((\d+), (\d+)\)/\((\d+), (\d+)\)/(\d+) stop: \((\d+), (\d+)\)/\((\d+), (\d+)\)/(\d+)', line) + + durable_start_start = int(matches.group(1)) + durable_start_end = int(matches.group(2)) + self.durable_start = Timestamp(durable_start_start, durable_start_end) + start_start = int(matches.group(3)) + start_end = int(matches.group(4)) + self.start = Timestamp(start_start, start_end) + self.start_txn = int(matches.group(5)) + + durable_stop_start = int(matches.group(6)) + durable_stop_end = int(matches.group(7)) + self.durable_stop = Timestamp(durable_start_start, durable_start_end) + stop_start = int(matches.group(8)) + stop_end = int(matches.group(9)) + self.stop = Timestamp(start_start, start_end) + self.stop_txn = int(matches.group(10)) + def __init_hs_update_restored(self, line): - self.type = OpType.HS_UPDATE_VALID + self.type = OpType.HS_UPDATE_RESTORED self.file = self.__extract_file(line) matches = re.search('txnid=(\d+)', line)