-
Type:
Build Failure
-
Resolution: Fixed
-
Priority:
Major - P3
-
Affects Version/s: None
-
Component/s: Test Format, Truncate
-
Storage Engines, Storage Engines - Foundations
-
62.446
-
None
-
None
-
65
format-stress-test-disagg-switch-1 on amazon2023-disagg-stress
Host: i-038563e8ea02776ce
Project: wiredtiger-disagg
Commit: 71cc6aae
Please refer to BF(G) Playbook for instructions on handling BF and BFG tickets as well as Auto-Resolution Rules
Task Logs:
format-stress-test-disagg-switch-1 task_log
Logs:
Task logger initialized (agent version '2026-05-18' from Evergreen build revision 'b18c97ea9e7b925a0fb4f02254f3cb9b0faaf014').
Starting task 'wiredtiger_disagg_amazon2023_disagg_stress_format_stress_test_disagg_switch_1_71cc6aae1cca77adb71f702270c0eac1dc3bcb59_26_05_20_04_31_27', execution 0.
Running pre-task commands.
Running command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre'.
Finished command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre' in 2.730219ms.
Running command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre'.
Finished command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre' in 40.459µs.
Finished running pre-task commands in 2.986524ms.
Running task commands.
Running command 's3.get' in function 'fetch artifacts' (step 1 of 2).
Fetching remote file 'wiredtiger/amazon2023-disagg-stress/71cc6aae1cca77adb71f702270c0eac1dc3bcb59/artifacts/compile_wiredtiger_disagg_amazon2023_disagg_stress_71cc6aae1cca77adb71f702270c0eac1dc3bcb59_26_05_20_04_31_27.tgz' from S3 bucket 'build_external' (attempt 1 of 5).
Finished command 's3.get' in function 'fetch artifacts' (step 1 of 2) in 5.370452982s.
Running command 'shell.exec' in function 'format test disagg' (step 2 of 2).
export WT_TOPDIR=$(git rev-parse --show-toplevel)
export WT_BUILDDIR=$WT_TOPDIR/cmake_build
if [ "$OS" = "Windows_NT" ]; then
export PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH
else
export PATH=/opt/mongodbtoolchain/v5/bin:$PATH
export LD_LIBRARY_PATH=$WT_BUILDDIR
fi
# Create the common sanitizer options and export the specific sanitizer environment
# variables.
COMMON_SAN_OPTIONS="abort_on_error=1:disable_coredump=0"
if [[ "" =~ ASan ]]; then
export ASAN_OPTIONS="$COMMON_SAN_OPTIONS:unmap_shadow_on_exit=1"
asan_lib=$(clang -print-file-name=libclang_rt.asan.so)
[ -f "$asan_lib" ] || { echo "ERROR: libclang_rt.asan.so not found by clang"; exit 1; }
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(dirname "$asan_lib")"
export TESTUTIL_BYPASS_ASAN=1
elif [[ "" =~ MSan ]]; then
export MSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
export TESTUTIL_MSAN=1
elif [[ "" =~ TSan ]]; then
export TSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
export TESTUTIL_TSAN=1
elif [[ "" =~ UBSan ]]; then
export UBSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_stacktrace=1"
export TESTUTIL_UBSAN=1
fi
if [[ 1 -eq 1 ]]; then
# The preloaded library should be downloaded/built during "configure wiredtiger" step.
# DO NOT MOVE:This variable must be set in the same function(shell) that runs a test,
# and as close as possible to the test execution, since it affects all the binaries.
export LD_PRELOAD=$WT_TOPDIR/TCMALLOC_LIB/libtcmalloc.so
if [ ! -f "$LD_PRELOAD" ]; then
echo "Error: TCMalloc support was requested, but the library doesn't exist: $LD_PRELOAD"
exit 1
fi
fi
# Set extensions to load
EXT="extensions=["
EXT="$EXT ext/compressors/snappy/libwiredtiger_snappy.so,"
EXT="$EXT ext/collators/reverse/libwiredtiger_reverse_collator.so,"
EXT="$EXT ext/encryptors/rotn/libwiredtiger_rotn.so,"
EXT="$EXT ext/page_log/palite/libwiredtiger_palite.so,"
EXT="$EXT ]"
export EXT
Dump Environment
eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
BASH_FUNC_which%%=() { ( alias;
CI=true
EVR_AGENT_PID=3578
EVR_TASK_ID=wiredtiger_disagg_amazon2023_disagg_stress_format_stress_test_disagg_switch_1_71cc6aae1cca77adb71f702270c0eac1dc3bcb59_26_05_20_04_31_27
EXT=extensions=[ ext/compressors/snappy/libwiredtiger_snappy.so, ext/collators/reverse/libwiredtiger_reverse_collator.so, ext/encryptors/rotn/libwiredtiger_rotn.so, ext/page_log/palite/libwiredtiger_palite.so, ]
GOCACHE=/data/mci/e5926558f0c9cc0634903dee58f702ce/wiredtiger/cmake_build/test/format/.gocache
GOTRACEBACK=none
HISTCONTROL=ignoredups
HISTSIZE=1000
HOME=/home/ec2-user
HOSTNAME=ip-10-122-59-84.ec2.internal
INVOCATION_ID=b4bf0ee9364f4d9eb72293b2f8e80740
JASPER_ID=980e7281-75e1-4d74-a3a9-9b60350f2655
JASPER_MANAGER=488caec7-9e1a-44a8-879d-8ead1787d70b
JOURNAL_STREAM=8:2459
LANG=C.UTF-8
LC_ALL=C
LD_LIBRARY_PATH=/data/mci/e5926558f0c9cc0634903dee58f702ce/wiredtiger/cmake_build
LD_PRELOAD=/data/mci/e5926558f0c9cc0634903dee58f702ce/wiredtiger/TCMALLOC_LIB/libtcmalloc.so
LESSOPEN=||/usr/bin/lesspipe.sh %s
LOGNAME=ec2-user
MAIL=/var/spool/mail/ec2-user
PATH=/opt/mongodbtoolchain/v5/bin:/usr/sbin:/usr/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/node/bin:/opt/node/bin:/opt/node/bin
PWD=/data/mci/e5926558f0c9cc0634903dee58f702ce/wiredtiger/cmake_build/test/format
SHELL=/bin/bash
SHLVL=1
SYSTEMD_COLORS=false
SYSTEMD_EXEC_PID=3358
S_COLORS=auto
TEMP=/data/mci/e5926558f0c9cc0634903dee58f702ce/tmp
TMP=/data/mci/e5926558f0c9cc0634903dee58f702ce/tmp
TMPDIR=/data/mci/e5926558f0c9cc0634903dee58f702ce/tmp
USER=ec2-user
WT_BUILDDIR=/data/mci/e5926558f0c9cc0634903dee58f702ce/wiredtiger/cmake_build
WT_TOPDIR=/data/mci/e5926558f0c9cc0634903dee58f702ce/wiredtiger
_=/usr/bin/printenv
which_declare=declare -f
}
echo "Dump Environment"
printenv | sort
# Fail, show the configuration file.
fail() {
echo "======= FAILURE =========="
[ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG
exit 1
}
for i in $(seq 5); do
echo Iteration $i/5
rm -rf RUNDIR
./t -c ../../../test/format/CONFIG.disagg disagg.mode=switch runs.timer=10:15 ops.prepare=1 || fail
if [ true = true ]; then
./t -R $format_args || fail
fi
done
t: WARNING: table.3 skipped verify because of EBUSY
t: WARNING: table.1 skipped verify because of EBUSY
t: WARNING: table.2 skipped verify because of EBUSY
Iteration 1/5
t: process 3850 running
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=2236943,metadata_checksum=a83afd87,database_size=462265500,version=1,compatible_version=1],timestamp=0x3b303b) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=3827390,metadata_checksum=a71cd1,database_size=451280286,version=1,compatible_version=1],timestamp=0x5af93f) ---
--- [Follower] Picked up checkpoint (metadata=[metadata_lsn=5265384,metadata_checksum=482f9e76,database_size=442252481,version=1,compatible_version=1],timestamp=0x79017c) ---
t: successful run completed (976 seconds)
t: WARNING: table.3 skipped verify because of EBUSY
t: WARNING: turning off ops.truncate to work with ops.prepare and precise checkpoint
format-stress-test-disagg-switch-1 task_log
Logs:
[1779257306:257673][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_DEFAULT][ERROR]: __timestamp_no_ts_fix, 497: WiredTiger assertion failed: 'select_tw->stop_txn >= select_tw->start_txn'. Expression returned false [1779257306:257696][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_DEFAULT][ERROR]: __wt_abort, 29: aborting WiredTiger library [1779257306:257709][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257715][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257717][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257719][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257721][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257723][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257725][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257727][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257728][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257730][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257732][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257734][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257736][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257737][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257739][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_btcur_prev, 706: Error at src/btree/bt_curprev.c:706: "WT_NOTFOUND" failed: WT_NOTFOUND: item not found [1779257306:257742][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_prev, 246: Error at src/cursor/cur_file.c:246: "ret" failed: WT_NOTFOUND: item not found [1779257306:257744][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near, 873: Error at src/cursor/cur_hs.c:873: "ret" failed: WT_NOTFOUND: item not found [1779257306:257748][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257750][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __wt_btcur_prev, 706: Error at src/btree/bt_curprev.c:706: "WT_NOTFOUND" failed: WT_NOTFOUND: item not found [1779257306:257752][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curfile_prev, 246: Error at src/cursor/cur_file.c:246: "ret" failed: WT_NOTFOUND: item not found [1779257306:257754][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near, 873: Error at src/cursor/cur_hs.c:873: "ret" failed: WT_NOTFOUND: item not found [1779257306:257756][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257758][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257759][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257761][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257763][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257765][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257766][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257768][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257772][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257774][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257776][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257778][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257780][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257781][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257783][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257785][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257787][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][ERROR]: __curhs_search_near_helper, 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1779257306:257789][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_V
format-stress-test-disagg-switch-1 task_log
Logs:
t: process 4533 running --- [Follower] Picked up checkpoint (metadata=[metadata_lsn=5471707,metadata_checksum=97588ce,database_size=438216488,version=1,compatible_version=1],timestamp=0x7c3d7b) --- 0x50377fc95800:[1779257306:257701][4533:0xffff869debc0], t, file:T00001.wt_stable, eviction-server: [WT_VERB_ERROR_RETURNS][WARNING]: 21100 errors occurred, only the last 100 are shown ======= FAILURE ========== ############################################ # RUN PARAMETERS: V3 ############################################ assert.read_timestamp=0 background_compact=0 background_compact.free_space_target=82 backup=0 backup.incremental=off backup.incr_granularity=15557 backup.live_restore=0 backup.live_restore_read_size=16384 backup.live_restore_threads=1 block_cache=0 block_cache.cache_on_checkpoint=0 block_cache.cache_on_writes=1 block_cache.size=73 cache=3072 cache.evict_max=1 cache.eviction_dirty_target=0 cache.eviction_dirty_trigger=0 cache.eviction_updates_target=0 cache.eviction_updates_trigger=0 cache.minimum=0 cache.maximum=0 checkpoint=on checkpoint.log_size=188 checkpoint.wait=69 checkpoint_threads=3 compact.free_space_target=12 debug.background_compact=0 debug.checkpoint_retention=2 debug.cursor_reposition=0 debug.eviction=0 debug.log_retention=6 debug.realloc_exact=0 debug.realloc_malloc=0 debug.slow_checkpoint=0 debug.table_logging=0 debug.update_restore_evict=0 disagg.internal_page_delta=1 disagg.leaf_page_delta=1 disagg.multi=0 disagg.multi_validation=0 disagg.enabled=1 disagg.layered=1 disagg.mode=switch disagg.page_log=palite disagg.key_provider=0 disagg.page_log.verbose=0 disagg.drain_threads=16 disagg.preserve=0 disk.data_extend=0 disk.encryption=none disk.mmap=1 disk.mmap_all=0 eviction.evict_use_softptr=0 file_manager.close_handle_minimum=21 file_manager.close_idle_time=7 file_manager.close_scan_interval=7 format.abort=0 format.independent_thread_rng=1 format.major_timeout=0 import=0 logging=0 logging.compression=none logging.file_max=413662 logging.prealloc=0 logging.remove=0 obsolete_cleanup.method=off obsolete_cleanup.wait=1464 ops.alter=0 ops.compaction=0 ops.hs_cursor=1 ops.bound_cursor=0 ops.prepare=1 ops.reserve=18 ops.random_cursor=0 ops.salvage=0 ops.throttle=0 ops.throttle.sleep_us=171222 ops.verify=1 prefetch=1 prefetch.default=0 precise_checkpoint=1 preserve_prepared=0 quiet=1 random.data_seed=8649186 random.extra_seed=7492855 rollback_to_stable_threads=8 runs.in_memory=0 runs.ops=0 runs.predictable_replay=0 runs.source=layered runs.tables=3 runs.threads=4 runs.timer=10 runs.type=row-store runs.verify_failure_dump=0 statistics.mode=fast statistics_log.sources=off stress.aggressive_stash_free=0 stress.aggressive_sweep=0 stress.checkpoint=0 stress.checkpoint_evict_page=0 stress.checkpoint_prepare=0 stress.compact_slow=0 stress.evict_reposition=0 stress.failpoint_eviction_split=0 stress.failpoint_hs_delete_key_from_ts=0 stress.failpoint_rec_before_wrapup=0 stress.hs_checkpoint_delay=0 stress.hs_search=0 stress.hs_sweep=0 stress.prefetch_delay=0 stress.prepare_resolution_1=0 stress.sleep_before_read_overflow_onpage=0 stress.split_1=0 stress.split_2=0 stress.split_3=0 stress.split_4=0 stress.split_5=0 stress.split_6=1 stress.split_7=0 stress.split_8=0 tiered_storage.flush_frequency=0 tiered_storage.storage_source=off transaction.implicit=0 transaction.operation_timeout_ms=2000 transaction.timestamps=1 wiredtiger.config=off wiredtiger.rwlock=0 wiredtiger.leak_memory=0 ############################################ # TABLE PARAMETERS: table 1 ############################################ table1.btree.compression=none table1.btree.dictionary=0 table1.btree.internal_key_truncation=1 table1.btree.internal_page_max=11 table1.btree.key_max=74 table1.btree.key_min=26 table1.btree.leaf_page_max=13 table1.btree.memory_page_max=2 table1.btree.prefix_len=0 table1.btree.prefix_compression=1 table1.btree.prefix_compression_min=4 table1.btree.reverse=0 table1.btree.split_pct=59 table1.btree.value_max=1379 table1.btree.value_min=10 table1.disk.checksum=unencrypted table1.disk.firstfit=0 table1.ops.pareto=0 table1.ops.pareto.skew=31 table1.ops.pct.delete=65 table1.ops.pct.insert=5 table1.ops.pct.modify=10 table1.ops.pct.read=5 table1.ops.pct.write=15 table1.ops.truncate=1 table1.runs.mirror=1 table1.runs.rows=856224 ############################################ # TABLE PARAMETERS: table 2 ############################################ table2.btree.compression=none table2.btree.dictionary=1 table2.btree.internal_key_truncation=1 table2.btree.internal_page_max=16 table2.btree.key_max=30 table2.btree.key_min=25 table2.btree.leaf_page_max=9 table2.btree.memory_page_max=9 table2.btree.prefix_len=0 table2.btree.prefix_compression=1 table2.btree.prefix_compression_min=5 table2.btree.reverse=0 table2.btree.split_pct=69 table2.btree.value_max=2171 table2.btree.value_min=0 table2.disk.checksum=on table2.disk.firstfit=0 table2.ops.pareto=0 table2.ops.pareto.skew=30 table2.ops.pct.delete=5 table2.ops.pct.insert=43 table2.ops.pct.modify=24 table2.ops.pct.read=3 table2.ops.pct.write=25 table2.ops.truncate=1 table2.runs.mirror=1 table2.runs.rows=856224 ############################################ # TABLE PARAMETERS: table 3 ############################################ table3.btree.compression=none table3.btree.dictionary=0 table3.btree.internal_key_truncation=1 table3.btree.internal_page_max=17 table3.btree.key_max=31 table3.btree.key_min=25 table3.btree.leaf_page_max=10 table3.btree.memory_page_max=2 table3.btree.prefix_len=0 table3.btree.prefix_compression=0 table3.btree.prefix_compression_min=2 table3.btree.reverse=0 table3.btree.split_pct=84 table3.btree.value_max=1926 table3.btree.value_min=12 table3.disk.checksum=off table3.disk.firstfit=0 table3.ops.pareto=0 table3.ops.pareto.skew=32 table3.ops.pct.delete=15 table3.ops.pct.insert=5 table3.ops.pct.modify=18 table3.ops.pct.read=2 table3.ops.pct.write=60 table3.ops.truncate=1 table3.runs.mirror=0 table3.runs.rows=824384 bash: line 67: 4533 Aborted (core dumped) ./t -R $format_args Command 'shell.exec' in function 'format test disagg' (step 2 of 2) failed: shell script encountered problem: exit code 1. Finished command 'shell.exec' in function 'format test disagg' (step 2 of 2) in 17m30.772902736s. Running task commands failed: running command: command failed: shell script encountered problem: exit code 1 Finished running task commands in 17m36.143733648s. Task completed - FAILURE. Running post-task commands.
format-stress-test-disagg-switch-1 task_log
Logs:
#0 0x0000ffff894c4454 in __pthread_kill_implementation () from /lib64/libc.so.6 #0 0x0000ffff894c4454 in __pthread_kill_implementation () from /lib64/libc.so.6 #1 0x0000ffff8947b320 [PAC] in raise () from /lib64/libc.so.6 #2 0x0000ffff89462224 [PAC] in abort () from /lib64/libc.so.6 #3 0x0000ffff8984c550 [PAC] in __wt_abort (session=session@entry=0x50377fc95800) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/os_common/os_abort.c:32 #4 0x0000ffff8988b448 in __timestamp_no_ts_fix (session=session@entry=0x50377fc95800, select_tw=select_tw@entry=0xffff869dc0e8) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/reconcile/rec_visibility.c:497 #5 0x0000ffff8988d574 in __wti_rec_upd_select (session=session@entry=0x50377fc95800, r=r@entry=0x5037b063e580, ins=ins@entry=0x0, rip=rip@entry=0x5037e4537508, vpack=vpack@entry=0xffff869dc158, upd_select=upd_select@entry=0xffff869dc0d8) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/reconcile/rec_visibility.c:1650 #6 0x0000ffff89882eb0 in __wti_rec_row_leaf (session=session@entry=0x50377fc95800, r=r@entry=0x5037b063e580, pageref=pageref@entry=0x503827018000, salvage=salvage@entry=0x0) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/reconcile/rec_row.c:1138 #7 0x0000ffff8989a6f8 in __reconcile (session=session@entry=0x50377fc95800, ref=ref@entry=0x503827018000, salvage=salvage@entry=0x0, flags=flags@entry=672, page_lockedp=page_lockedp@entry=0xffff869dcb2f) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/reconcile/rec_write.c:316 #8 0x0000ffff8989982c in __wt_reconcile (session=session@entry=0x50377fc95800, ref=ref@entry=0x503827018000, salvage=salvage@entry=0x0, flags=flags@entry=672) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/reconcile/rec_write.c:127 #9 0x0000ffff897fd878 in __evict_reconcile (session=session@entry=0x50377fc95800, ref=ref@entry=0x503827018000, evict_flags=evict_flags@entry=0) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/evict/evict_page.c:1277 #10 0x0000ffff897fffbc in __wt_evict (session=session@entry=0x50377fc95800, ref=0x503827018000, previous_state=3 '\003', flags=flags@entry=0) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/evict/evict_page.c:443 #11 0x0000ffff897f5064 in __wti_evict_page (session=session@entry=0x50377fc95800, is_server=is_server@entry=true) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/evict/evict_dispatch.c:254 #12 0x0000ffff89800c9c in __wti_evict_lru_pages (session=session@entry=0x50377fc95800, is_server=is_server@entry=true) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/evict/evict_queue.c:140 #13 0x0000ffff89804660 in __evict_pass (session=session@entry=0x50377fc95800) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/evict/evict_thread.c:695 #14 0x0000ffff898047a0 in __evict_server (session=session@entry=0x50377fc95800, did_work=did_work@entry=0xffff869dde9f) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/evict/evict_thread.c:198 #15 0x0000ffff89804c38 in __evict_thread_run (session=0x50377fc95800, thread=0x50377fe06960) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/evict/evict_thread.c:93 #16 0x0000ffff89913a14 in __thread_run (arg=0x50377fe06960) at /data/mci/4fdbfd06c72ceb9b94724677fc456b1c/wiredtiger/src/support/thread_group.c:32 #17 0x0000ffff894c2834 in start_thread () from /lib64/libc.so.6 #18 0x0000ffff89466e5c [PAC] in thread_start () from /lib64/libc.so.6