(disagg.mode=leader) test/format key OOO (non-delta) block_cache=1

XMLWordPrintableJSON

    • Storage Engines, Storage Engines - Persistence
    • SE Persistence - 2025-10-24
    • 8

      There is a key OOO without non-delta configuration and reproduces consistently with the given config.

      [2025/09/30 12:52:48.304] fail() {
      [2025/09/30 12:52:48.304]   echo "======= FAILURE =========="
      [2025/09/30 12:52:48.304]   [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG
      [2025/09/30 12:52:48.304]   exit 1
      [2025/09/30 12:52:48.304] }
      [2025/09/30 12:52:48.304] for i in $(seq 5); do
      [2025/09/30 12:52:48.304]   echo Iteration $i/5
      [2025/09/30 12:52:48.304]   rm -rf RUNDIR
      [2025/09/30 12:52:48.304]   ./t -c ../../../test/format/CONFIG.disagg disagg.mode=leader ops.verify=1 runs.mirror=1 table1.runs.source=table table1.disagg.enabled=0 runs.tables=2 runs.rows=100000 runs.ops=300000  || fail
      [2025/09/30 12:53:11.421] Iteration 1/5
      [2025/09/30 12:53:11.421] t: process 4405 running
      [2025/09/30 12:58:53.173]   ./t -R $format_args  || fail
      [2025/09/30 12:58:53.173] done
      [2025/09/30 12:58:53.173] t: WARNING: table.1 skipped verify because of EBUSY
      [2025/09/30 12:58:56.602] table.2 skipped verify because verify does not support disagg delta pages. table.3 skipped verify because verify does not support disagg delta pages. table.2 skipped verify because verify does not support disagg delta pages. table.3 skipped verify because verify does not support disagg delta pages. t: successful run completed (368 seconds)
      [2025/09/30 12:58:56.823]  t: process 4737 running
      [2025/09/30 13:00:19.501] [1759201219:501818][4737:0xffff99642040], t, file:T00003.wt_stable, WT_CURSOR.next: [WT_VERB_OUT_OF_ORDER][ERROR]: WT_CURSOR.next out-of-order returns: returned key 0000058604.09/opqrstu then key 0000058560.01/opqrstu
      

      CONFIG:

      ############################################
      #  RUN PARAMETERS: V3
      ############################################
      assert.read_timestamp=0
      background_compact=0
      background_compact.free_space_target=65
      backup=0
      backup.incremental=off
      backup.incr_granularity=765
      backup.live_restore=0
      backup.live_restore_read_size=32
      backup.live_restore_threads=0
      block_cache=1
      block_cache.cache_on_checkpoint=1
      block_cache.cache_on_writes=1
      block_cache.size=821
      cache=4104
      cache.evict_max=0
      cache.eviction_dirty_target=0
      cache.eviction_dirty_trigger=0
      cache.eviction_updates_target=0
      cache.eviction_updates_trigger=0
      cache.minimum=0
      cache.maximum=0
      checkpoint=on
      checkpoint.log_size=117
      checkpoint.wait=82
      chunk_cache=0
      chunk_cache.capacity=1043
      chunk_cache.chunk_size=3
      chunk_cache.storage_path=off
      chunk_cache.type=off
      compact.free_space_target=22
      debug.background_compact=0
      debug.checkpoint_retention=10
      debug.cursor_reposition=0
      debug.eviction=0
      debug.log_retention=1
      debug.realloc_exact=0
      debug.realloc_malloc=0
      debug.slow_checkpoint=0
      debug.table_logging=0
      debug.update_restore_evict=0
      disagg.page_log=palm
      disagg.mode=leader
      disagg.enabled=1
      disagg.layered=1
      disk.data_extend=0
      disk.encryption=none
      disk.mmap=1
      disk.mmap_all=1
      eviction.evict_use_softptr=0
      file_manager.close_handle_minimum=32
      file_manager.close_idle_time=18
      file_manager.close_scan_interval=9
      format.abort=0
      format.independent_thread_rng=1
      format.major_timeout=0
      import=0
      logging=0
      logging.compression=none
      logging.file_max=443272
      logging.prealloc=1
      logging.remove=1
      obsolete_cleanup.method=off
      obsolete_cleanup.wait=1473
      ops.alter=0
      ops.compaction=0
      ops.hs_cursor=0
      ops.pct.modify=0
      ops.bound_cursor=0
      ops.prepare=0
      ops.random_cursor=0
      ops.salvage=0
      ops.throttle=0
      ops.throttle.sleep_us=676624
      ops.truncate=0
      ops.verify=1
      prefetch=1
      precise_checkpoint=1
      preserve_prepared=0
      quiet=1
      random.data_seed=5640004
      random.extra_seed=3018997
      runs.in_memory=0
      runs.mirror=1
      runs.ops=300000
      runs.predictable_replay=0
      runs.rows=100000
      runs.source=layered
      runs.tables=2
      runs.threads=19
      runs.timer=360
      runs.type=row-store
      runs.verify_failure_dump=0
      statistics.mode=all
      statistics_log.sources=off
      stress.aggressive_stash_free=0
      stress.aggressive_sweep=0
      stress.checkpoint=0
      stress.checkpoint_evict_page=0
      stress.checkpoint_prepare=0
      stress.compact_slow=0
      stress.evict_reposition=0
      stress.failpoint_eviction_split=0
      stress.failpoint_hs_delete_key_from_ts=0
      stress.failpoint_rec_before_wrapup=0
      stress.hs_checkpoint_delay=0
      stress.hs_search=0
      stress.hs_sweep=0
      stress.prefetch_delay=0
      stress.prepare_resolution_1=0
      stress.sleep_before_read_overflow_onpage=0
      stress.split_1=0
      stress.split_2=0
      stress.split_3=0
      stress.split_4=0
      stress.split_5=0
      stress.split_6=0
      stress.split_7=0
      stress.split_8=0
      tiered_storage.flush_frequency=0
      tiered_storage.storage_source=off
      transaction.implicit=0
      transaction.operation_timeout_ms=2000
      transaction.timestamps=1
      wiredtiger.config=off
      wiredtiger.rwlock=1
      wiredtiger.leak_memory=0
      ############################################
      #  TABLE PARAMETERS: table 1
      ############################################
      table1.btree.compression=none
      table1.btree.dictionary=0
      table1.btree.internal_key_truncation=1
      table1.btree.internal_page_max=17
      table1.btree.key_max=37
      table1.btree.key_min=21
      table1.btree.leaf_page_max=12
      table1.btree.memory_page_max=7
      table1.btree.prefix_len=0
      table1.btree.prefix_compression=1
      table1.btree.prefix_compression_min=3
      table1.btree.reverse=0
      table1.btree.split_pct=51
      table1.btree.value_max=1450
      table1.btree.value_min=19
      table1.disagg.enabled=0
      table1.disk.checksum=unencrypted
      table1.disk.firstfit=0
      table1.ops.pareto=0
      table1.ops.pareto.skew=97
      table1.ops.pct.delete=94
      table1.ops.pct.insert=3
      table1.ops.pct.read=3
      table1.ops.pct.write=0
      table1.runs.source=table
      ############################################
      #  TABLE PARAMETERS: table 2
      ############################################
      table2.btree.compression=none
      table2.btree.dictionary=0
      table2.btree.internal_key_truncation=1
      table2.btree.internal_page_max=17
      table2.btree.key_max=56
      table2.btree.key_min=18
      table2.btree.leaf_page_max=17
      table2.btree.memory_page_max=5
      table2.btree.prefix_len=0
      table2.btree.prefix_compression=1
      table2.btree.prefix_compression_min=7
      table2.btree.reverse=0
      table2.btree.split_pct=54
      table2.btree.value_max=357
      table2.btree.value_min=14
      table2.disk.checksum=on
      table2.disk.firstfit=0
      table2.ops.pareto=0
      table2.ops.pareto.skew=47
      table2.ops.pct.delete=28
      table2.ops.pct.insert=6
      table2.ops.pct.read=58
      table2.ops.pct.write=8
      ############################################
      #  TABLE PARAMETERS: table 3
      ############################################
      table3.btree.compression=none
      table3.btree.dictionary=0
      table3.btree.internal_key_truncation=1
      table3.btree.internal_page_max=13
      table3.btree.key_max=32
      table3.btree.key_min=13
      table3.btree.leaf_page_max=11
      table3.btree.memory_page_max=6
      table3.btree.prefix_len=0
      table3.btree.prefix_compression=1
      table3.btree.prefix_compression_min=5
      table3.btree.reverse=0
      table3.btree.split_pct=82
      table3.btree.value_max=2976
      table3.btree.value_min=5
      table3.disk.checksum=off
      table3.disk.firstfit=1
      table3.ops.pareto=0
      table3.ops.pareto.skew=17
      table3.ops.pct.delete=3
      table3.ops.pct.insert=94
      table3.ops.pct.read=1
      table3.ops.pct.write=2
      

            Assignee:
            [DO NOT USE] Backlog - Storage Engines Team
            Reporter:
            Jie Chen
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

              Created:
              Updated:
              Resolved: