Summary

In disaggregated multi-node testing the leader and follower databases diverge: the two nodes return different results for the same operation, and the run then aborts on a whole-database checksum mismatch.

Symptom

While running the disaggregated multi-node format stress tests (format-stress-test-disagg-multi-*), the follower and the leader disagree on the result of an operation:

• A MODIFY on the follower returns WT_NOTFOUND (-31803) while the leader returns 0 for the same key at the same read timestamp.
• At the end of the run this manifests as a whole-database checksum mismatch (leader_hash != follower_hash), which aborts both nodes in disagg_sync_multi_node (test/format/format_disagg.c:196).

Deletes and truncates are disabled in the workload.

Reproduction

• Branch: develop (observed on 7ee03cba2c and neighbouring commits).
• The disagg multi-node validation tasks, on both the stress and ASAN disagg stress variants.
• The following CONFIG reproduces reliably (about 50% of multi-node tasks fail):

  • ############################################
    #  RUN PARAMETERS: V3
    ############################################
    assert.read_timestamp=0
    background_compact=0
    background_compact.free_space_target=40
    backup=0
    backup.incremental=off
    backup.incr_granularity=10259
    backup.live_restore=0
    backup.live_restore_read_size=8192
    backup.live_restore_threads=9
    block_cache=0
    block_cache.cache_on_checkpoint=1
    block_cache.cache_on_writes=0
    block_cache.size=44
    btree.reverse=0
    cache=3072
    cache.evict_max=5
    cache.eviction_dirty_target=0
    cache.eviction_dirty_trigger=0
    cache.eviction_updates_target=0
    cache.eviction_updates_trigger=0
    cache.minimum=0
    cache.maximum=0
    checkpoint=on
    checkpoint.log_size=146
    checkpoint.wait=75
    checkpoint_threads=1
    compact.free_space_target=78
    debug.background_compact=0
    debug.checkpoint_retention=2
    debug.cursor_reposition=0
    debug.disagg_slow_truncate_follower=0
    debug.eviction=0
    debug.log_retention=7
    debug.realloc_exact=0
    debug.realloc_malloc=0
    debug.slow_checkpoint=0
    debug.slow_truncate=0
    debug.table_logging=0
    debug.update_restore_evict=0
    disagg.internal_page_delta=1
    disagg.leaf_page_delta=1
    disagg.multi=1
    disagg.multi_validation=1
    disagg.enabled=1
    disagg.layered=1
    disagg.mode=leader
    disagg.page_log=palite
    disagg.key_provider=2
    disagg.page_log.verbose=0
    disagg.drain_threads=11
    disagg.preserve=1
    disagg.stepdown_async=0
    disk.data_extend=0
    disk.encryption=rotn-7
    disk.mmap=1
    disk.mmap_all=0
    eviction.evict_use_softptr=0
    file_manager.close_handle_minimum=14
    file_manager.close_idle_time=33
    file_manager.close_scan_interval=26
    format.abort=0
    format.independent_thread_rng=0
    format.major_timeout=0
    import=0
    logging=0
    logging.compression=none
    logging.file_max=191044
    logging.prealloc=1
    logging.remove=0
    obsolete_cleanup.method=off
    obsolete_cleanup.wait=3133
    ops.alter=0
    ops.compaction=0
    ops.hs_cursor=1
    ops.pct.delete=0
    ops.bound_cursor=0
    ops.prepare=0
    ops.reserve=1
    ops.random_cursor=1
    ops.salvage=0
    ops.throttle=0
    ops.throttle.sleep_us=124755
    ops.verify=1
    prefetch=1
    prefetch.default=0
    precise_checkpoint=1
    preserve_prepared=1
    quiet=1
    random.data_seed=5947916
    random.extra_seed=15179692
    rollback_to_stable_threads=10
    runs.in_memory=0
    runs.ops=500000
    runs.predictable_replay=1
    runs.source=layered
    runs.tables=3
    runs.threads=13
    runs.timer=360
    runs.type=row-store
    runs.verify_failure_dump=0
    statistics.mode=fast
    statistics_log.sources=off
    stress.aggressive_stash_free=0
    stress.aggressive_sweep=0
    stress.checkpoint=0
    stress.checkpoint_evict_page=0
    stress.checkpoint_prepare=0
    stress.compact_slow=0
    stress.evict_reposition=0
    stress.failpoint_eviction_split=1
    stress.failpoint_hs_delete_key_from_ts=0
    stress.failpoint_rec_before_wrapup=0
    stress.hs_checkpoint_delay=0
    stress.hs_search=0
    stress.hs_sweep=0
    stress.prefetch_delay=0
    stress.prepare_resolution_1=0
    stress.sleep_before_read_overflow_onpage=0
    stress.split_1=1
    stress.split_2=0
    stress.split_3=0
    stress.split_4=0
    stress.split_5=0
    stress.split_6=0
    stress.split_7=0
    stress.split_8=0
    tiered_storage.flush_frequency=0
    tiered_storage.storage_source=off
    transaction.implicit=0
    transaction.operation_timeout_ms=2000
    transaction.timestamps=1
    wiredtiger.config=off
    wiredtiger.rwlock=1
    wiredtiger.leak_memory=0
    ############################################
    #  TABLE PARAMETERS: table 1
    ############################################
    table1.btree.compression=none
    table1.btree.dictionary=0
    table1.btree.internal_key_truncation=1
    table1.btree.internal_page_max=9
    table1.btree.key_max=120
    table1.btree.key_min=19
    table1.btree.leaf_page_max=14
    table1.btree.memory_page_max=6
    table1.btree.prefix_len=0
    table1.btree.prefix_compression=1
    table1.btree.prefix_compression_min=8
    table1.btree.split_pct=56
    table1.btree.value_max=2946
    table1.btree.value_min=12
    table1.disk.checksum=off
    table1.disk.firstfit=0
    table1.ops.pareto=0
    table1.ops.pareto.skew=16
    table1.ops.pct.insert=7
    table1.ops.pct.modify=10
    table1.ops.pct.read=58
    table1.ops.pct.write=25
    table1.ops.truncate=0
    table1.runs.mirror=0
    table1.runs.rows=427354
    ############################################
    #  TABLE PARAMETERS: table 2
    ############################################
    table2.btree.compression=none
    table2.btree.dictionary=0
    table2.btree.internal_key_truncation=1
    table2.btree.internal_page_max=11
    table2.btree.key_max=60
    table2.btree.key_min=17
    table2.btree.leaf_page_max=13
    table2.btree.memory_page_max=5
    table2.btree.prefix_len=0
    table2.btree.prefix_compression=1
    table2.btree.prefix_compression_min=0
    table2.btree.split_pct=66
    table2.btree.value_max=2151
    table2.btree.value_min=10
    table2.disk.checksum=unencrypted
    table2.disk.firstfit=0
    table2.ops.pareto=0
    table2.ops.pareto.skew=30
    table2.ops.pct.insert=5
    table2.ops.pct.modify=0
    table2.ops.pct.read=89
    table2.ops.pct.write=6
    table2.ops.truncate=0
    table2.runs.mirror=0
    table2.runs.rows=862867
    ############################################
    #  TABLE PARAMETERS: table 3
    ############################################
    table3.btree.compression=none
    table3.btree.dictionary=0
    table3.btree.internal_key_truncation=1
    table3.btree.internal_page_max=16
    table3.btree.key_max=128
    table3.btree.key_min=27
    table3.btree.leaf_page_max=12
    table3.btree.memory_page_max=9
    table3.btree.prefix_len=0
    table3.btree.prefix_compression=1
    table3.btree.prefix_compression_min=2
    table3.btree.split_pct=80
    table3.btree.value_max=2975
    table3.btree.value_min=18
    table3.disk.checksum=unencrypted
    table3.disk.firstfit=0
    table3.ops.pareto=0
    table3.ops.pareto.skew=13
    table3.ops.pct.insert=3
    table3.ops.pct.modify=78
    table3.ops.pct.read=19
    table3.ops.pct.write=0
    table3.ops.truncate=0
    table3.runs.mirror=0
    table3.runs.rows=491757