Task logger initialized (agent version '2026-05-01a' from Evergreen build revision 'ff8929f8fff3db7854eb32fdc24fd98e3029ed6a').
Starting task 'wiredtiger_disagg_amazon2023_disagg_stress_format_stress_test_disagg_switch_1_9f1d4c2946cb7ba11640a7d1b67c1a6f4ad3df34_26_05_05_23_52_05', execution 0.
Running pre-task commands.
Running command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre'.
Finished command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre' in 2.401152ms.
Running command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre'.
Finished command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre' in 35.116µs.
Finished running pre-task commands in 2.620827ms.
Running task commands.
Running command 's3.get' in function 'fetch artifacts' (step 1 of 2).
Fetching remote file 'wiredtiger/amazon2023-disagg-stress/9f1d4c2946cb7ba11640a7d1b67c1a6f4ad3df34/artifacts/compile_wiredtiger_disagg_amazon2023_disagg_stress_9f1d4c2946cb7ba11640a7d1b67c1a6f4ad3df34_26_05_05_23_52_05.tgz' from S3 bucket 'build_external' (attempt 1 of 5).
Finished command 's3.get' in function 'fetch artifacts' (step 1 of 2) in 5.270141568s.
Running command 'shell.exec' in function 'format test disagg' (step 2 of 2).
export WT_TOPDIR=$(git rev-parse --show-toplevel)
export WT_BUILDDIR=$WT_TOPDIR/cmake_build
if [ "$OS" = "Windows_NT" ]; then
  export PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH
else
  export PATH=/opt/mongodbtoolchain/v5/bin:$PATH
  export LD_LIBRARY_PATH=$WT_BUILDDIR
fi
# Create the common sanitizer options and export the specific sanitizer environment
# variables.
COMMON_SAN_OPTIONS="abort_on_error=1:disable_coredump=0"
if [[ "" =~ ASan ]]; then
  export ASAN_OPTIONS="$COMMON_SAN_OPTIONS:unmap_shadow_on_exit=1"
  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(dirname $(/opt/mongodbtoolchain/v5/bin/clang -print-file-name=libclang_rt.asan.so))"
  export TESTUTIL_BYPASS_ASAN=1
elif [[ "" =~ MSan ]]; then
  export MSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
  export TESTUTIL_MSAN=1
elif [[ "" =~ TSan ]]; then
  export TSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
  export TESTUTIL_TSAN=1
elif [[ "" =~ UBSan ]]; then
  export UBSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_stacktrace=1"
  export TESTUTIL_UBSAN=1
fi
if [[ 1 -eq 1 ]]; then
  # The preloaded library should be downloaded/built during "configure wiredtiger" step.
  # DO NOT MOVE:This variable must be set in the same function(shell) that runs a test,
  # and as close as possible to the test execution, since it affects all the binaries.
  export LD_PRELOAD=$WT_TOPDIR/TCMALLOC_LIB/libtcmalloc.so
  if [ ! -f "$LD_PRELOAD" ]; then
    echo "Error: TCMalloc support was requested, but the library doesn't exist: $LD_PRELOAD"
    exit 1
  fi
fi
# Set extensions to load
EXT="extensions=["
EXT="$EXT ext/compressors/snappy/libwiredtiger_snappy.so,"
EXT="$EXT ext/collators/reverse/libwiredtiger_reverse_collator.so,"
EXT="$EXT ext/encryptors/rotn/libwiredtiger_rotn.so,"
EXT="$EXT ext/page_log/palite/libwiredtiger_palite.so,"
EXT="$EXT ]"
export EXT
Dump Environment
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
BASH_FUNC_which%%=() {  ( alias;
CI=true
EVR_AGENT_PID=3618
EVR_TASK_ID=wiredtiger_disagg_amazon2023_disagg_stress_format_stress_test_disagg_switch_1_9f1d4c2946cb7ba11640a7d1b67c1a6f4ad3df34_26_05_05_23_52_05
EXT=extensions=[ ext/compressors/snappy/libwiredtiger_snappy.so, ext/collators/reverse/libwiredtiger_reverse_collator.so, ext/encryptors/rotn/libwiredtiger_rotn.so, ext/page_log/palite/libwiredtiger_palite.so, ]
GOCACHE=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/wiredtiger/cmake_build/test/format/.gocache
GOTRACEBACK=none
HISTCONTROL=ignoredups
HISTSIZE=1000
HOME=/home/ec2-user
HOSTNAME=ip-10-122-62-111.ec2.internal
INVOCATION_ID=78f8a0b6689946da8f2502e78c3b140b
JASPER_ID=f2df3f1f-2762-4e43-a2cf-f0e1635594b6
JASPER_MANAGER=a99d6b77-6269-4264-ae5b-e5fb7b711b40
JOURNAL_STREAM=8:28698
LANG=C.UTF-8
LC_ALL=C
LD_LIBRARY_PATH=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/wiredtiger/cmake_build
LD_PRELOAD=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/wiredtiger/TCMALLOC_LIB/libtcmalloc.so
LESSOPEN=||/usr/bin/lesspipe.sh %s
LOGNAME=ec2-user
MAIL=/var/spool/mail/ec2-user
PATH=/opt/mongodbtoolchain/v5/bin:/usr/sbin:/usr/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/node/bin:/opt/node/bin:/opt/node/bin
PWD=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/wiredtiger/cmake_build/test/format
SHELL=/bin/bash
SHLVL=1
SYSTEMD_COLORS=false
SYSTEMD_EXEC_PID=3415
S_COLORS=auto
TEMP=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/tmp
TMP=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/tmp
TMPDIR=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/tmp
USER=ec2-user
WT_BUILDDIR=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/wiredtiger/cmake_build
WT_TOPDIR=/data/mci/e0825c595366d4ec3f5dccaa29e0b95d/wiredtiger
_=/usr/bin/printenv
which_declare=declare -f
}
echo "Dump Environment"
printenv | sort
# Fail, show the configuration file.
fail() {
  echo "======= FAILURE =========="
  [ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG
  exit 1
}
for i in $(seq 5); do
  echo Iteration $i/5
  rm -rf RUNDIR
  ./t -c ../../../test/format/CONFIG.disagg disagg.mode=switch runs.timer=10:15 ops.prepare=1  || fail
  if [ true = true ]; then
    ./t -R $format_args  || fail
  fi
done
t: WARNING: table.3 skipped verify because of EBUSY
Iteration 1/5
t: process 3891 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=1714826,metadata_checksum=5e0d50bb,database_size=452096175,version=1,compatible_version=1],timestamp=0x1561103) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=3149038,metadata_checksum=471523e5,database_size=462612840,version=1,compatible_version=1],timestamp=0x25919d0) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=4321276,metadata_checksum=8a3a4230,database_size=442457865,version=1,compatible_version=1],timestamp=0x331c165) ---
t: successful run completed (930 seconds)
t: WARNING: table.2 skipped verify because of EBUSY
t: WARNING: turning off ops.random_cursor with disagg as they are currently problematic and can cause stalls
t: WARNING: turning off ops.truncate to work with ops.prepare and precise checkpoint
 t: process 4569 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=4430057,metadata_checksum=d11d07d2,database_size=452118012,version=1,compatible_version=1],timestamp=0x3404813) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=5533392,metadata_checksum=2a8c2f4f,database_size=432318442,version=1,compatible_version=1],timestamp=0x404f6e0) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=6453365,metadata_checksum=248d645e,database_size=422457774,version=1,compatible_version=1],timestamp=0x4ac72fe) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=7441042,metadata_checksum=90c2c747,database_size=405674198,version=1,compatible_version=1],timestamp=0x5649e3f) ---
t: successful run completed (1060 seconds)
t: WARNING: table selection didn't support mirroring, turning off mirroring
t: WARNING: table.3 skipped verify because of EBUSY
t: WARNING: table.1 skipped verify because of EBUSY
t: WARNING: table.2 skipped verify because of EBUSY
 Iteration 2/5
t: process 5297 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=1515253,metadata_checksum=b14272fc,database_size=205537689,version=1,compatible_version=1],timestamp=0xd902af) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=2648994,metadata_checksum=b869be6b,database_size=198003745,version=1,compatible_version=1],timestamp=0x187aa36) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=3598092,metadata_checksum=cb44e837,database_size=211697424,version=1,compatible_version=1],timestamp=0x225ee3d) ---
t: successful run completed (839 seconds)
t: WARNING: turning off ops.random_cursor with disagg as they are currently problematic and can cause stalls
t: WARNING: turning off ops.truncate to work with ops.prepare and precise checkpoint

[1778032625:461412][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_turtle_read, 653: Error at src/meta/meta_turtle.c:653: "WT_NOTFOUND" failed: WT_NOTFOUND: item not found
[1778032625:461437][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_search, 315: Error at src/cursor/cur_file.c:315: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461441][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_metadata_search, 341: Error at src/meta/meta_table.c:341: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461443][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_search, 315: Error at src/cursor/cur_file.c:315: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461446][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_metadata_search, 341: Error at src/meta/meta_table.c:341: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461447][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_search, 315: Error at src/cursor/cur_file.c:315: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461449][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_metadata_search, 341: Error at src/meta/meta_table.c:341: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461451][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_btcur_next, 795: Error at src/btree/bt_curnext.c:795: "WT_NOTFOUND" failed: WT_NOTFOUND: item not found
[1778032625:461454][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_next, 186: Error at src/cursor/cur_file.c:186: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461455][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_btcur_next, 795: Error at src/btree/bt_curnext.c:795: "WT_NOTFOUND" failed: WT_NOTFOUND: item not found
[1778032625:461457][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_next, 186: Error at src/cursor/cur_file.c:186: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461459][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_btcur_next, 795: Error at src/btree/bt_curnext.c:795: "WT_NOTFOUND" failed: WT_NOTFOUND: item not found
[1778032625:461461][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_next, 186: Error at src/cursor/cur_file.c:186: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461463][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_search, 315: Error at src/cursor/cur_file.c:315: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461465][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_metadata_search, 341: Error at src/meta/meta_table.c:341: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461467][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_search, 315: Error at src/cursor/cur_file.c:315: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461468][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_metadata_remove, 281: Error at src/meta/meta_table.c:281: "cursor->search(cursor)" failed: WT_NOTFOUND: item not found
[1778032625:461471][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __drop_file, 56: Error at src/schema/schema_drop.c:56: "__wt_metadata_remove(session, uri)" failed: WT_NOTFOUND: item not found
[1778032625:461473][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __drop_file, 73: Error at src/schema/schema_drop.c:73: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461475][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_search, 315: Error at src/cursor/cur_file.c:315: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461477][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_metadata_search, 341: Error at src/meta/meta_table.c:341: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461479][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_btcur_next, 795: Error at src/btree/bt_curnext.c:795: "WT_NOTFOUND" failed: WT_NOTFOUND: item not found
[1778032625:461481][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_next, 186: Error at src/cursor/cur_file.c:186: "ret" failed: WT_NOTFOUND: item not found
[1778032625:461483][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_next, 315: Error at src/cursor/cur_hs.c:315: "__curhs_file_cursor_next(session, file_cursor)" failed: WT_NOTFOUND: item not found
[1778032625:461485][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_DEFAULT][ERROR]: __hs_verify_id, 77: the associated history store key 0000079881.00/opqrstuvwx was not found in the data store file:T00001.wt_stable: WT_PANIC: WiredTiger library panic
[1778032625:461487][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_DEFAULT][ERROR]: __hs_verify_id, 77: the process must exit and restart: WT_PANIC: WiredTiger library panic
[1778032625:461491][5889:0xffff94763040], t, file:T00001.wt_stable, verify hs: [WT_VERB_DEFAULT][ERROR]: __wt_abort, 29: aborting WiredTiger library

 t: process 5889 running
======= FAILURE ==========
############################################
#  RUN PARAMETERS: V3
############################################
assert.read_timestamp=0
background_compact=0
background_compact.free_space_target=19
backup=0
backup.incremental=off
backup.incr_granularity=13745
backup.live_restore=0
backup.live_restore_read_size=128
backup.live_restore_threads=4
block_cache=0
block_cache.cache_on_checkpoint=0
block_cache.cache_on_writes=0
block_cache.size=12
cache=3072
cache.evict_max=5
cache.eviction_dirty_target=0
cache.eviction_dirty_trigger=0
cache.eviction_updates_target=0
cache.eviction_updates_trigger=0
cache.minimum=0
cache.maximum=0
checkpoint=on
checkpoint.log_size=129
checkpoint.wait=44
checkpoint_threads=3
compact.free_space_target=60
debug.background_compact=0
debug.checkpoint_retention=6
debug.cursor_reposition=0
debug.eviction=0
debug.log_retention=1
debug.realloc_exact=0
debug.realloc_malloc=1
debug.slow_checkpoint=0
debug.table_logging=0
debug.update_restore_evict=0
disagg.internal_page_delta=1
disagg.leaf_page_delta=1
disagg.multi=0
disagg.multi_validation=0
disagg.enabled=1
disagg.layered=1
disagg.mode=switch
disagg.page_log=palite
disagg.key_provider=0
disagg.page_log.verbose=0
disagg.drain_threads=4
disagg.preserve=0
disk.data_extend=0
disk.encryption=none
disk.mmap=0
disk.mmap_all=0
eviction.evict_use_softptr=0
file_manager.close_handle_minimum=22
file_manager.close_idle_time=34
file_manager.close_scan_interval=15
format.abort=0
format.independent_thread_rng=0
format.major_timeout=0
import=0
logging=0
logging.compression=none
logging.file_max=64806
logging.prealloc=1
logging.remove=1
obsolete_cleanup.method=off
obsolete_cleanup.wait=1585
ops.alter=0
ops.compaction=0
ops.hs_cursor=1
ops.bound_cursor=0
ops.prepare=1
ops.reserve=16
ops.random_cursor=0
ops.salvage=0
ops.throttle=0
ops.throttle.sleep_us=754810
ops.verify=1
prefetch=0
prefetch.default=0
precise_checkpoint=1
preserve_prepared=1
quiet=1
random.data_seed=4609272
random.extra_seed=9865218
rollback_to_stable_threads=6
runs.in_memory=0
runs.ops=0
runs.predictable_replay=0
runs.source=layered
runs.tables=3
runs.threads=12
runs.timer=11
runs.type=row-store
runs.verify_failure_dump=0
statistics.mode=fast
statistics_log.sources=off
stress.aggressive_stash_free=0
stress.aggressive_sweep=0
stress.checkpoint=0
stress.checkpoint_evict_page=0
stress.checkpoint_prepare=0
stress.compact_slow=0
stress.evict_reposition=0
stress.failpoint_eviction_split=0
stress.failpoint_hs_delete_key_from_ts=0
stress.failpoint_rec_before_wrapup=0
stress.hs_checkpoint_delay=0
stress.hs_search=0
stress.hs_sweep=0
stress.prefetch_delay=0
stress.prepare_resolution_1=0
stress.sleep_before_read_overflow_onpage=0
stress.split_1=0
stress.split_2=0
stress.split_3=0
stress.split_4=0
stress.split_5=0
stress.split_6=0
stress.split_7=0
stress.split_8=0
tiered_storage.flush_frequency=0
tiered_storage.storage_source=off
transaction.implicit=0
transaction.operation_timeout_ms=2000
transaction.timestamps=1
wiredtiger.config=off
wiredtiger.rwlock=0
wiredtiger.leak_memory=0
############################################
#  TABLE PARAMETERS: table 1
############################################
table1.btree.compression=none
table1.btree.dictionary=0
table1.btree.internal_key_truncation=1
table1.btree.internal_page_max=9
table1.btree.key_max=79
table1.btree.key_min=15
table1.btree.leaf_page_max=9
table1.btree.memory_page_max=1
table1.btree.prefix_len=0
table1.btree.prefix_compression=1
table1.btree.prefix_compression_min=2
table1.btree.reverse=0
table1.btree.split_pct=98
table1.btree.value_max=135
table1.btree.value_min=1
table1.disk.checksum=on
table1.disk.firstfit=0
table1.ops.pareto=0
table1.ops.pareto.skew=58
table1.ops.pct.delete=0
table1.ops.pct.insert=0
table1.ops.pct.modify=0
table1.ops.pct.read=0
table1.ops.pct.write=100
table1.ops.truncate=1
table1.runs.mirror=1
table1.runs.rows=100374
############################################
#  TABLE PARAMETERS: table 2
############################################
table2.btree.compression=none
table2.btree.dictionary=0
table2.btree.internal_key_truncation=1
table2.btree.internal_page_max=10
table2.btree.key_max=125
table2.btree.key_min=15
table2.btree.leaf_page_max=10
table2.btree.memory_page_max=8
table2.btree.prefix_len=0
table2.btree.prefix_compression=0
table2.btree.prefix_compression_min=5
table2.btree.reverse=0
table2.btree.split_pct=71
table2.btree.value_max=96
table2.btree.value_min=12
table2.disk.checksum=off
table2.disk.firstfit=0
table2.ops.pareto=0
table2.ops.pareto.skew=21
table2.ops.pct.delete=5
table2.ops.pct.insert=14
table2.ops.pct.modify=2
table2.ops.pct.read=56
table2.ops.pct.write=23
table2.ops.truncate=1
table2.runs.mirror=1
table2.runs.rows=100374
############################################
#  TABLE PARAMETERS: table 3
############################################
table3.btree.compression=none
table3.btree.dictionary=0
table3.btree.internal_key_truncation=1
table3.btree.internal_page_max=14
table3.btree.key_max=22
table3.btree.key_min=22
table3.btree.leaf_page_max=13
table3.btree.memory_page_max=1
table3.btree.prefix_len=0
table3.btree.prefix_compression=0
table3.btree.prefix_compression_min=5
table3.btree.reverse=0
table3.btree.split_pct=65
table3.btree.value_max=466
table3.btree.value_min=7
table3.disk.checksum=on
table3.disk.firstfit=0
table3.ops.pareto=0
table3.ops.pareto.skew=14
table3.ops.pct.delete=0
table3.ops.pct.insert=2
table3.ops.pct.modify=29
table3.ops.pct.read=0
table3.ops.pct.write=69
table3.ops.truncate=1
table3.runs.mirror=0
table3.runs.rows=869057
bash: line 65:  5889 Aborted                 (core dumped) ./t -R $format_args
Command 'shell.exec' in function 'format test disagg' (step 2 of 2) failed: shell script encountered problem: exit code 1.
Finished command 'shell.exec' in function 'format test disagg' (step 2 of 2) in 47m11.949686418s.
Running task commands failed: running command: command failed: shell script encountered problem: exit code 1
Finished running task commands in 47m17.220163417s.
Task completed - FAILURE.
Running post-task commands.

#0  0x0000ffff92ec4454 in __pthread_kill_implementation () from /lib64/libc.so.6
#0  0x0000ffff92ec4454 in __pthread_kill_implementation () from /lib64/libc.so.6
#1  0x0000ffff92e7b320 [PAC] in raise () from /lib64/libc.so.6
#2  0x0000ffff92e62224 [PAC] in abort () from /lib64/libc.so.6
#3  0x0000ffff93245f50 [PAC] in __wt_abort (session=session@entry=0x53dcbfc98e40) at /data/mci/5fea7ff3eee984dd1f605980ef517c3d/wiredtiger/src/os_common/os_abort.c:32
#4  0x0000ffff932efc30 in __wt_panic_func (session=session@entry=0x53dcbfc98e40, error=error@entry=-31804, func=func@entry=0xffff933ee248 <__PRETTY_FUNCTION__.15> "__hs_verify_id", line=line@entry=77, category=category@entry=WT_VERB_DEFAULT, fmt=fmt@entry=0xffff9338cdb0 "the associated history store key %s was not found in the data store %s") at /data/mci/5fea7ff3eee984dd1f605980ef517c3d/wiredtiger/src/support/err.c:633
#5  0x0000ffff932090c0 in __hs_verify_id (session=session@entry=0x53dcbfc98e40, hs_cursor=0x53dcbfc8c500, ds_cbt=0x53dcbf66e800, this_btree_id=25) at /data/mci/5fea7ff3eee984dd1f605980ef517c3d/wiredtiger/src/history/hs_verify.c:77
#6  0x0000ffff9320978c in __hs_verify (session=session@entry=0x53dcbfc98e40, hs_id=<optimized out>) at /data/mci/5fea7ff3eee984dd1f605980ef517c3d/wiredtiger/src/history/hs_verify.c:236
#7  0x0000ffff93209d00 in __wt_hs_verify (session=0x53dcbfc98e40) at /data/mci/5fea7ff3eee984dd1f605980ef517c3d/wiredtiger/src/history/hs_verify.c:277
#8  0x0000ffff93152f10 in wiredtiger_open (home=home@entry=0x463f48 <g+120> "RUNDIR", event_handler=event_handler@entry=0x461ce0 <event_handler>, config=config@entry=0xfffffb338968 ",error_prefix=\"t\",statistics=(fast),statistics_log=(json,on_close,wait=5),timing_stress_for_test=[],file_manager=[,close_handle_minimum=22,close_idle_time=34,close_scan_interval=15],debug_mode=[,check"..., connectionp=connectionp@entry=0xfffffb338d68) at /data/mci/5fea7ff3eee984dd1f605980ef517c3d/wiredtiger/src/conn/conn_api.c:3636
#9  0x0000000000421b2c in wts_open (home=home@entry=0x463f48 <g+120> "RUNDIR", connp=connp@entry=0x463ed0 <g>, verify_metadata=verify_metadata@entry=true) at /data/mci/5fea7ff3eee984dd1f605980ef517c3d/wiredtiger/test/format/wts.c:818
#10 0x000000000041b838 in main (argc=<optimized out>, argv=<optimized out>) at /data/mci/5fea7ff3eee984dd1f605980ef517c3d/wiredtiger/test/format/t.c:345