Task logger initialized (agent version '2026-04-23' from Evergreen build revision 'c0c879e910217bcf5d1a107102646feec2053002').
Starting task 'wiredtiger_disagg_amazon2023_disagg_asan_stress_format_stress_test_disagg_switch_1_5fd6c5a42ba10406887ef3cc6b13ad2d6f3d0a83_26_04_27_23_50_12', execution 0.
Running pre-task commands.
Running command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre'.
Finished command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre' in 2.71372ms.
Running command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre'.
Finished command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre' in 44.789µs.
Finished running pre-task commands in 2.989514ms.
Running task commands.
Running command 's3.get' in function 'fetch artifacts' (step 1 of 2).
Fetching remote file 'wiredtiger/amazon2023-disagg-asan-stress/5fd6c5a42ba10406887ef3cc6b13ad2d6f3d0a83/artifacts/compile_wiredtiger_disagg_amazon2023_disagg_asan_stress_5fd6c5a42ba10406887ef3cc6b13ad2d6f3d0a83_26_04_27_23_50_12.tgz' from S3 bucket 'build_external' (attempt 1 of 5).
Finished command 's3.get' in function 'fetch artifacts' (step 1 of 2) in 7.702779995s.
Running command 'shell.exec' in function 'format test disagg' (step 2 of 2).
export WT_TOPDIR=$(git rev-parse --show-toplevel)
export WT_BUILDDIR=$WT_TOPDIR/cmake_build
if [ "$OS" = "Windows_NT" ]; then
  export PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH
else
  export PATH=/opt/mongodbtoolchain/v5/bin:$PATH
  export LD_LIBRARY_PATH=$WT_BUILDDIR
fi
# Create the common sanitizer options and export the specific sanitizer environment
# variables.
COMMON_SAN_OPTIONS="abort_on_error=1:disable_coredump=0"
if [[ "-DCMAKE_BUILD_TYPE=ASan" =~ ASan ]]; then
  export ASAN_OPTIONS="$COMMON_SAN_OPTIONS:unmap_shadow_on_exit=1"
  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(dirname $(/opt/mongodbtoolchain/v5/bin/clang -print-file-name=libclang_rt.asan.so))"
  export TESTUTIL_BYPASS_ASAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ MSan ]]; then
  export MSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
  export TESTUTIL_MSAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ TSan ]]; then
  export TSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
  export TESTUTIL_TSAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ UBSan ]]; then
  export UBSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_stacktrace=1"
  export TESTUTIL_UBSAN=1
fi
if [[ 0 -eq 1 ]]; then
  # The preloaded library should be downloaded/built during "configure wiredtiger" step.
  # DO NOT MOVE:This variable must be set in the same function(shell) that runs a test,
  # and as close as possible to the test execution, since it affects all the binaries.
  export LD_PRELOAD=$WT_TOPDIR/TCMALLOC_LIB/libtcmalloc.so
  if [ ! -f "$LD_PRELOAD" ]; then
    echo "Error: TCMalloc support was requested, but the library doesn't exist: $LD_PRELOAD"
    exit 1
  fi
fi
# Set extensions to load
EXT="extensions=["
EXT="$EXT ext/compressors/snappy/libwiredtiger_snappy.so,"
EXT="$EXT ext/collators/reverse/libwiredtiger_reverse_collator.so,"
EXT="$EXT ext/encryptors/rotn/libwiredtiger_rotn.so,"
EXT="$EXT ext/page_log/palite/libwiredtiger_palite.so,"
EXT="$EXT ]"
export EXT
export LSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_suppressions=0:suppressions=$(git rev-parse --show-toplevel)/test/evergreen/asan_leaks.supp"
Dump Environment
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
ASAN_OPTIONS=abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
BASH_FUNC_which%%=() {  ( alias;
CI=true
EVR_AGENT_PID=3694
EVR_TASK_ID=wiredtiger_disagg_amazon2023_disagg_asan_stress_format_stress_test_disagg_switch_1_5fd6c5a42ba10406887ef3cc6b13ad2d6f3d0a83_26_04_27_23_50_12
EXT=extensions=[ ext/compressors/snappy/libwiredtiger_snappy.so, ext/collators/reverse/libwiredtiger_reverse_collator.so, ext/encryptors/rotn/libwiredtiger_rotn.so, ext/page_log/palite/libwiredtiger_palite.so, ]
GOCACHE=/data/mci/a53685f640822a68147d3add2aee743d/wiredtiger/cmake_build/test/format/.gocache
GOTRACEBACK=none
HISTCONTROL=ignoredups
HISTSIZE=1000
HOME=/home/ec2-user
HOSTNAME=ip-10-122-58-121.ec2.internal
INVOCATION_ID=b22b29fb7505465995d35f94033f5d86
JASPER_ID=334c74bf-45bd-4be7-8f10-5c35d922a7b6
JASPER_MANAGER=6068182b-285a-4c31-9c0d-8c02a614bb74
JOURNAL_STREAM=8:444
LANG=C.UTF-8
LC_ALL=C
LD_LIBRARY_PATH=/data/mci/a53685f640822a68147d3add2aee743d/wiredtiger/cmake_build:/opt/mongodbtoolchain/revisions/e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd/stow/llvm-v5.NkZ/lib/clang/19/lib/aarch64-mongodb-linux
LESSOPEN=||/usr/bin/lesspipe.sh %s
LOGNAME=ec2-user
LSAN_OPTIONS=abort_on_error=1:disable_coredump=0:print_suppressions=0:suppressions=/data/mci/a53685f640822a68147d3add2aee743d/wiredtiger/test/evergreen/asan_leaks.supp
MAIL=/var/spool/mail/ec2-user
PATH=/opt/mongodbtoolchain/v5/bin:/usr/sbin:/usr/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/node/bin:/opt/node/bin:/opt/node/bin
PWD=/data/mci/a53685f640822a68147d3add2aee743d/wiredtiger/cmake_build/test/format
SHELL=/bin/bash
SHLVL=1
SYSTEMD_COLORS=false
SYSTEMD_EXEC_PID=3562
S_COLORS=auto
TEMP=/data/mci/a53685f640822a68147d3add2aee743d/tmp
TESTUTIL_BYPASS_ASAN=1
TMP=/data/mci/a53685f640822a68147d3add2aee743d/tmp
TMPDIR=/data/mci/a53685f640822a68147d3add2aee743d/tmp
USER=ec2-user
WT_BUILDDIR=/data/mci/a53685f640822a68147d3add2aee743d/wiredtiger/cmake_build
WT_TOPDIR=/data/mci/a53685f640822a68147d3add2aee743d/wiredtiger
_=/usr/bin/printenv
which_declare=declare -f
}
echo "Dump Environment"
printenv | sort
export ASAN_OPTIONS="$COMMON_SAN_OPTIONS:unmap_shadow_on_exit=1"
# Fail, show the configuration file.
fail() {
  echo "======= FAILURE =========="
  [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG
  exit 1
}
for i in $(seq 5); do
  echo Iteration $i/5
  rm -rf RUNDIR
  ./t -c ../../../test/format/CONFIG.disagg disagg.mode=switch runs.timer=10:15  || fail
  if [ true = true ]; then
Iteration 1/5
t: process 3897 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=132803,metadata_checksum=e3363fd1,database_size=212309547,version=1,compatible_version=1],timestamp=0x80ee87) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=238836,metadata_checksum=1a51a9af,database_size=214644533,version=1,compatible_version=1],timestamp=0x10cab46) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=346764,metadata_checksum=a708eb74,database_size=196989994,version=1,compatible_version=1],timestamp=0x19a444b) ---
t: successful run completed (1048 seconds)
    ./t -R $format_args  || fail
  fi
done
t: WARNING: turning off ops.random_cursor with disagg as they are currently problematic and can cause stalls
t: WARNING: table selection didn't support mirroring, turning off mirroring
t: WARNING: table.1 skipped verify because of EBUSY
 t: process 4766 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=444389,metadata_checksum=c76e455e,database_size=210176146,version=1,compatible_version=1],timestamp=0x2250577) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=536786,metadata_checksum=19f89f45,database_size=191975690,version=1,compatible_version=1],timestamp=0x2b8d8e7) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=624164,metadata_checksum=51c38155,database_size=196764885,version=1,compatible_version=1],timestamp=0x34d14c0) ---
t: successful run completed (1078 seconds)
 Iteration 2/5
t: process 5629 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=69532,metadata_checksum=acf27792,database_size=150140042,version=1,compatible_version=1],timestamp=0x9a4115) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=157975,metadata_checksum=41fc5528,database_size=169310375,version=1,compatible_version=1],timestamp=0x131c71f) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=253329,metadata_checksum=14af1fc1,database_size=167194315,version=1,compatible_version=1],timestamp=0x1c8955d) ---
t: successful run completed (651 seconds)
t: WARNING: turning off ops.random_cursor with disagg as they are currently problematic and can cause stalls
 t: process 6256 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=253329,metadata_checksum=14af1fc1,database_size=167194315,version=1,compatible_version=1],timestamp=0x1c8955d) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=352749,metadata_checksum=a901cd6a,database_size=168411075,version=1,compatible_version=1],timestamp=0x256fa8f) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=440798,metadata_checksum=998f9988,database_size=170199300,version=1,compatible_version=1],timestamp=0x2e4f8c3) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=515140,metadata_checksum=63c17c32,database_size=171553782,version=1,compatible_version=1],timestamp=0x372ad1f) ---
t: successful run completed (785 seconds)
 Iteration 3/5
t: process 6970 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=150069,metadata_checksum=8bd394a4,database_size=286097220,version=1,compatible_version=1],timestamp=0x395f6f) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=268526,metadata_checksum=8ae70f12,database_size=299430460,version=1,compatible_version=1],timestamp=0x726c35) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=391412,metadata_checksum=663dc795,database_size=297018267,version=1,compatible_version=1],timestamp=0xabc437) ---
t: successful run completed (722 seconds)
t: WARNING: table selection didn't support mirroring, turning off mirroring
t: WARNING: turning off ops.random_cursor with disagg as they are currently problematic and can cause stalls
 t: process 7621 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=391412,metadata_checksum=663dc795,database_size=297018267,version=1,compatible_version=1],timestamp=0xabc437) ---
--- [Follower] Pic

[1777344655:653490][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_DEFAULT][ERROR]: int __rec_hs_cursor_pos(WT_SESSION_IMPL *, WT_CURSOR *, uint32_t, const WT_ITEM *, wt_timestamp_t, WT_TIME_WINDOW *, _Bool *), 353: WiredTiger assertion failed: 'ts == 1 || ts == 0'. out-of-order timestamp update detected, found an existing update with hs_start_ts=(0, 12585896), time_window=start: durable_timestamp=(0, 0) timestamp=(0, 0) prepare_timestamp=(0, 0) prepared_id=0 transaction=0 | stop: durable_timestamp=(0, 0) timestamp=(4294967295, 4294967295) prepare_timestamp=(0, 0) prepared_id=0 transaction=18446744073709551605, that exists later than specified ts=(0, 12813315)
[1777344655:653677][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_DEFAULT][ERROR]: void __wt_abort(WT_SESSION_IMPL *), 29: aborting WiredTiger library

 t: process 8977 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=126635,metadata_checksum=c3ae0e5d,database_size=74879036,version=1,compatible_version=1],timestamp=0xc0134d) ---
0xffffb3a3a7c0:[1777344655:653727][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][WARNING]: 229 errors occurred, only the last 100 are shown

[1777344655:653790][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:653848][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:653899][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:653936][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:653969][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654002][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654038][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654075][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654114][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654148][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654183][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654224][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654256][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654311][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654346][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654382][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654417][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654453][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654494][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654529][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:654565][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654599][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654642][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654679][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654726][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654764][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654798][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654835][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654870][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654903][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654942][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_next(WT_CURSOR *), 321: Error at src/cursor/cur_hs.c:321: "__curhs_next_visible(session, hs_cursor)" failed: WT_NOTFOUND: item not found
[1777344655:654978][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found
[1777344655:655013][8977:0xffffaceb30e0], t, file:T00003.wt_stable, checkpoint-page-reconciliation-threads: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, W

======= FAILURE ==========
############################################
#  RUN PARAMETERS: V3
############################################
assert.read_timestamp=0
background_compact=0
background_compact.free_space_target=6
backup=0
backup.incremental=off
backup.incr_granularity=7026
backup.live_restore=0
backup.live_restore_read_size=1024
backup.live_restore_threads=7
block_cache=0
block_cache.cache_on_checkpoint=0
block_cache.cache_on_writes=0
block_cache.size=35
cache=3072
cache.evict_max=2
cache.eviction_dirty_target=0
cache.eviction_dirty_trigger=0
cache.eviction_updates_target=0
cache.eviction_updates_trigger=0
cache.minimum=0
cache.maximum=0
checkpoint=on
checkpoint.log_size=46
checkpoint.wait=90
checkpoint_threads=2
compact.free_space_target=16
debug.background_compact=0
debug.checkpoint_retention=3
debug.cursor_reposition=0
debug.eviction=0
debug.log_retention=7
debug.realloc_exact=0
debug.realloc_malloc=0
debug.slow_checkpoint=0
debug.table_logging=0
debug.update_restore_evict=0
disagg.internal_page_delta=1
disagg.leaf_page_delta=1
disagg.multi=0
disagg.multi_validation=0
disagg.enabled=1
disagg.layered=1
disagg.mode=switch
disagg.page_log=palite
disagg.key_provider=0
disagg.page_log.verbose=0
disagg.drain_threads=12
disagg.preserve=0
disk.data_extend=0
disk.encryption=none
disk.mmap=1
disk.mmap_all=0
eviction.evict_use_softptr=0
file_manager.close_handle_minimum=6
file_manager.close_idle_time=51
file_manager.close_scan_interval=19
format.abort=0
format.independent_thread_rng=1
format.major_timeout=0
import=0
logging=0
logging.compression=none
logging.file_max=503540
logging.prealloc=1
logging.remove=1
obsolete_cleanup.method=off
obsolete_cleanup.wait=157
ops.alter=0
ops.compaction=0
ops.hs_cursor=0
ops.pct.modify=0
ops.bound_cursor=0
ops.prepare=1
ops.reserve=5
ops.random_cursor=0
ops.salvage=0
ops.throttle=0
ops.throttle.sleep_us=99866
ops.verify=1
prefetch=1
precise_checkpoint=1
preserve_prepared=1
quiet=1
random.data_seed=11102255
random.extra_seed=863084
rollback_to_stable_threads=3
runs.in_memory=0
runs.ops=0
runs.predictable_replay=0
runs.source=layered
runs.tables=3
runs.threads=28
runs.timer=11
runs.type=row-store
runs.verify_failure_dump=0
statistics.mode=all
statistics_log.sources=off
stress.aggressive_stash_free=0
stress.aggressive_sweep=0
stress.checkpoint=0
stress.checkpoint_evict_page=0
stress.checkpoint_prepare=0
stress.compact_slow=0
stress.evict_reposition=0
stress.failpoint_eviction_split=0
stress.failpoint_hs_delete_key_from_ts=1
stress.failpoint_rec_before_wrapup=0
stress.hs_checkpoint_delay=0
stress.hs_search=0
stress.hs_sweep=0
stress.prefetch_delay=0
stress.prepare_resolution_1=0
stress.sleep_before_read_overflow_onpage=0
stress.split_1=0
stress.split_2=0
stress.split_3=0
stress.split_4=0
stress.split_5=0
stress.split_6=0
stress.split_7=0
stress.split_8=0
tiered_storage.flush_frequency=0
tiered_storage.storage_source=off
transaction.implicit=0
transaction.operation_timeout_ms=2000
transaction.timestamps=1
wiredtiger.config=off
wiredtiger.rwlock=0
wiredtiger.leak_memory=0
############################################
#  TABLE PARAMETERS: table 1
############################################
table1.btree.compression=zstd
table1.btree.dictionary=0
table1.btree.internal_key_truncation=1
table1.btree.internal_page_max=13
table1.btree.key_max=78
table1.btree.key_min=28
table1.btree.leaf_page_max=16
table1.btree.memory_page_max=1
table1.btree.prefix_len=0
table1.btree.prefix_compression=1
table1.btree.prefix_compression_min=7
table1.btree.reverse=0
table1.btree.split_pct=77
table1.btree.value_max=3829
table1.btree.value_min=9
table1.disk.checksum=unencrypted
table1.disk.firstfit=1
table1.ops.pareto=1
table1.ops.pareto.skew=3
table1.ops.pct.delete=6
table1.ops.pct.insert=32
table1.ops.pct.read=32
table1.ops.pct.write=30
table1.ops.truncate=1
table1.runs.mirror=1
table1.runs.rows=34024
############################################
#  TABLE PARAMETERS: table 2
############################################
table2.btree.compression=none
table2.btree.dictionary=0
table2.btree.internal_key_truncation=1
table2.btree.internal_page_max=14
table2.btree.key_max=126
table2.btree.key_min=15
table2.btree.leaf_page_max=14
table2.btree.memory_page_max=4
table2.btree.prefix_len=0
table2.btree.prefix_compression=1
table2.btree.prefix_compression_min=0
table2.btree.reverse=0
table2.btree.split_pct=83
table2.btree.value_max=2192
table2.btree.value_min=5
table2.disk.checksum=off
table2.disk.firstfit=0
table2.ops.pareto=0
table2.ops.pareto.skew=71
table2.ops.pct.delete=1
table2.ops.pct.insert=3
table2.ops.pct.read=1
table2.ops.pct.write=95
table2.ops.truncate=1
table2.runs.mirror=1
table2.runs.rows=34024
############################################
#  TABLE PARAMETERS: table 3
############################################
table3.btree.compression=zstd
table3.btree.dictionary=1
table3.btree.internal_key_truncation=1
table3.btree.internal_page_max=15
table3.btree.key_max=63
table3.btree.key_min=28
table3.btree.leaf_page_max=10
table3.btree.memory_page_max=1
table3.btree.prefix_len=76
table3.btree.prefix_compression=1
table3.btree.prefix_compression_min=0
table3.btree.reverse=0
table3.btree.split_pct=67
table3.btree.value_max=943
table3.btree.value_min=6
table3.disk.checksum=off
table3.disk.firstfit=0
table3.ops.pareto=1
table3.ops.pareto.skew=25
table3.ops.pct.delete=7
table3.ops.pct.insert=31
table3.ops.pct.read=18
table3.ops.pct.write=44
table3.ops.truncate=1
table3.runs.mirror=0
table3.runs.rows=932987
bash: line 66:  8977 Aborted                 (core dumped) ./t -R $format_args
Command 'shell.exec' in function 'format test disagg' (step 2 of 2) failed: shell script encountered problem: exit code 1.
Finished command 'shell.exec' in function 'format test disagg' (step 2 of 2) in 1h42m22.179220264s.
Running task commands failed: running command: command failed: shell script encountered problem: exit code 1
Finished running task commands in 1h42m29.882650909s.
Task completed - FAILURE.
Running post-task commands.