-
Type:
Build Failure
-
Resolution: Unresolved
-
Priority:
Major - P3
-
None
-
Affects Version/s: None
-
Component/s: Disagg CI-blocker
-
None
-
Storage Engines - Transactions
-
188.841
-
None
-
None
-
1
format-stress-test-disagg-switch-1 on amazon2023-disagg-asan-stress
Host: i-05489029d0c638aec
Project: wiredtiger-disagg
Commit: 74b747f6
Please refer to BF(G) Playbook for instructions on handling BF and BFG tickets as well as Auto-Resolution Rules
Task Logs:
format-stress-test-disagg-switch-1 task_log
Logs:
Task logger initialized (agent version '2026-05-01a' from Evergreen build revision '506cfa5dd480356b3052925f267bd0aabe7d8ec5').
Starting task 'wiredtiger_disagg_amazon2023_disagg_asan_stress_format_stress_test_disagg_switch_1_74b747f61a5ae66019571c662dccd609b921508f_26_05_07_23_02_39', execution 0.
Running pre-task commands.
Running command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre'.
Finished command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre' in 2.619431ms.
Running command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre'.
Finished command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre' in 42.492µs.
Finished running pre-task commands in 2.871815ms.
Running task commands.
Running command 's3.get' in function 'fetch artifacts' (step 1 of 2).
Fetching remote file 'wiredtiger/amazon2023-disagg-asan-stress/74b747f61a5ae66019571c662dccd609b921508f/artifacts/compile_wiredtiger_disagg_amazon2023_disagg_asan_stress_74b747f61a5ae66019571c662dccd609b921508f_26_05_07_23_02_39.tgz' from S3 bucket 'build_external' (attempt 1 of 5).
Finished command 's3.get' in function 'fetch artifacts' (step 1 of 2) in 18.008157575s.
Running command 'shell.exec' in function 'format test disagg' (step 2 of 2).
export WT_TOPDIR=$(git rev-parse --show-toplevel)
export WT_BUILDDIR=$WT_TOPDIR/cmake_build
if [ "$OS" = "Windows_NT" ]; then
export PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH
else
export PATH=/opt/mongodbtoolchain/v5/bin:$PATH
export LD_LIBRARY_PATH=$WT_BUILDDIR
fi
# Create the common sanitizer options and export the specific sanitizer environment
# variables.
COMMON_SAN_OPTIONS="abort_on_error=1:disable_coredump=0"
if [[ "-DCMAKE_BUILD_TYPE=ASan" =~ ASan ]]; then
export ASAN_OPTIONS="$COMMON_SAN_OPTIONS:unmap_shadow_on_exit=1"
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(dirname $(/opt/mongodbtoolchain/v5/bin/clang -print-file-name=libclang_rt.asan.so))"
export TESTUTIL_BYPASS_ASAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ MSan ]]; then
export MSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
export TESTUTIL_MSAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ TSan ]]; then
export TSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
export TESTUTIL_TSAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ UBSan ]]; then
export UBSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_stacktrace=1"
export TESTUTIL_UBSAN=1
fi
if [[ 0 -eq 1 ]]; then
# The preloaded library should be downloaded/built during "configure wiredtiger" step.
# DO NOT MOVE:This variable must be set in the same function(shell) that runs a test,
# and as close as possible to the test execution, since it affects all the binaries.
export LD_PRELOAD=$WT_TOPDIR/TCMALLOC_LIB/libtcmalloc.so
if [ ! -f "$LD_PRELOAD" ]; then
echo "Error: TCMalloc support was requested, but the library doesn't exist: $LD_PRELOAD"
exit 1
fi
fi
# Set extensions to load
EXT="extensions=["
EXT="$EXT ext/compressors/snappy/libwiredtiger_snappy.so,"
EXT="$EXT ext/collators/reverse/libwiredtiger_reverse_collator.so,"
EXT="$EXT ext/encryptors/rotn/libwiredtiger_rotn.so,"
EXT="$EXT ext/page_log/palite/libwiredtiger_palite.so,"
EXT="$EXT ]"
export EXT
export LSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_suppressions=0:suppressions=$(git rev-parse --show-toplevel)/test/evergreen/asan_leaks.supp"
Dump Environment
eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
ASAN_OPTIONS=abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
BASH_FUNC_which%%=() { ( alias;
CI=true
EVR_AGENT_PID=3603
EVR_TASK_ID=wiredtiger_disagg_amazon2023_disagg_asan_stress_format_stress_test_disagg_switch_1_74b747f61a5ae66019571c662dccd609b921508f_26_05_07_23_02_39
EXT=extensions=[ ext/compressors/snappy/libwiredtiger_snappy.so, ext/collators/reverse/libwiredtiger_reverse_collator.so, ext/encryptors/rotn/libwiredtiger_rotn.so, ext/page_log/palite/libwiredtiger_palite.so, ]
GOCACHE=/data/mci/f63802ed969efc5bcfde8b38b44defa0/wiredtiger/cmake_build/test/format/.gocache
GOTRACEBACK=none
HISTCONTROL=ignoredups
HISTSIZE=1000
HOME=/home/ec2-user
HOSTNAME=ip-10-122-61-168.ec2.internal
INVOCATION_ID=8a879be3df2f40f0855bb1c867e2eff6
JASPER_ID=e755ebfd-5b77-4ebe-a0c0-9b7465cb5649
JASPER_MANAGER=be1c46b8-14c9-4954-9ba2-23c993447371
JOURNAL_STREAM=8:23042
LANG=C.UTF-8
LC_ALL=C
LD_LIBRARY_PATH=/data/mci/f63802ed969efc5bcfde8b38b44defa0/wiredtiger/cmake_build:/opt/mongodbtoolchain/revisions/e921fc32d5c23d7cdb5cf406b05bf16eb5ab8dbd/stow/llvm-v5.NkZ/lib/clang/19/lib/aarch64-mongodb-linux
LESSOPEN=||/usr/bin/lesspipe.sh %s
LOGNAME=ec2-user
LSAN_OPTIONS=abort_on_error=1:disable_coredump=0:print_suppressions=0:suppressions=/data/mci/f63802ed969efc5bcfde8b38b44defa0/wiredtiger/test/evergreen/asan_leaks.supp
MAIL=/var/spool/mail/ec2-user
PATH=/opt/mongodbtoolchain/v5/bin:/usr/sbin:/usr/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/node/bin:/opt/node/bin:/opt/node/bin
PWD=/data/mci/f63802ed969efc5bcfde8b38b44defa0/wiredtiger/cmake_build/test/format
SHELL=/bin/bash
SHLVL=1
SYSTEMD_COLORS=false
SYSTEMD_EXEC_PID=3379
S_COLORS=auto
TEMP=/data/mci/f63802ed969efc5bcfde8b38b44defa0/tmp
TESTUTIL_BYPASS_ASAN=1
TMP=/data/mci/f63802ed969efc5bcfde8b38b44defa0/tmp
TMPDIR=/data/mci/f63802ed969efc5bcfde8b38b44defa0/tmp
USER=ec2-user
WT_BUILDDIR=/data/mci/f63802ed969efc5bcfde8b38b44defa0/wiredtiger/cmake_build
WT_TOPDIR=/data/mci/f63802ed969efc5bcfde8b38b44defa0/wiredtiger
_=/usr/bin/printenv
which_declare=declare -f
}
echo "Dump Environment"
printenv | sort
export ASAN_OPTIONS="$COMMON_SAN_OPTIONS:unmap_shadow_on_exit=1"
# Fail, show the configuration file.
fail() {
echo "======= FAILURE =========="
[ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG
exit 1
}
for i in $(seq 5); do
echo Iteration $i/5
rm -rf RUNDIR
./t -c ../../../test/format/CONFIG.disagg disagg.mode=switch runs.timer=10:15 ops.prepare=1 || fail
if [ true = true ]; then
./t -R $format_args || fail
fi
done
format-stress-test-disagg-switch-1 task_log
Logs:
[1778200061:405355][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_DEFAULT][ERROR]: int __curhs_insert(WT_CURSOR *), 1068: WiredTiger assertion failed: 'hs_cursor->time_window.stop_txn != ((18446744073709551615UL) - 10)'. Expression returned false [1778200061:405425][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_DEFAULT][ERROR]: void __wt_abort(WT_SESSION_IMPL *), 29: aborting WiredTiger library [1778200061:405438][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __wt_try_readlock(WT_SESSION_IMPL *, WT_RWLOCK *), 151: Error at src/support/mtx_rw.c:151: "__wt_atomic_cas_uint64_v(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY" failed: Device or resource busy [1778200061:405454][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405461][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405466][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405472][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405477][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __wt_try_readlock(WT_SESSION_IMPL *, WT_RWLOCK *), 151: Error at src/support/mtx_rw.c:151: "__wt_atomic_cas_uint64_v(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY" failed: Device or resource busy [1778200061:405482][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405488][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405493][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405498][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405503][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405508][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405513][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405518][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405523][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405528][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405533][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405538][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405543][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405548][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405553][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405558][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405563][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405568][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405573][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405578][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405583][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405588][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405593][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405598][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool), 745: Error at src/cursor/cur_hs.c:745: "cursor->search_near(cursor, &cmp)" failed: WT_NOTFOUND: item not found [1778200061:405603][3878:0xffff873380e0], t, file:WiredTigerSharedHS.wt_stable, WT_CURSOR.insert: [WT_VERB_ERROR_RETURNS][ERROR]: int __wt
format-stress-test-disagg-switch-1 task_log
Logs:
bash: line 66: 3878 Aborted (core dumped) ./t -c ../../../test/format/CONFIG.disagg disagg.mode=switch runs.timer=10:15 ops.prepare=1 Iteration 1/5 t: process 3878 running ======= FAILURE ========== ############################################ # RUN PARAMETERS: V3 ############################################ assert.read_timestamp=0 background_compact=0 background_compact.free_space_target=49 backup=0 backup.incremental=off backup.incr_granularity=5786 backup.live_restore=0 backup.live_restore_read_size=32 backup.live_restore_threads=9 block_cache=0 block_cache.cache_on_checkpoint=0 block_cache.cache_on_writes=1 block_cache.size=40 btree.reverse=0 cache=3072 cache.evict_max=3 cache.eviction_dirty_target=0 cache.eviction_dirty_trigger=0 cache.eviction_updates_target=0 cache.eviction_updates_trigger=0 cache.minimum=0 cache.maximum=0 checkpoint=on checkpoint.log_size=143 checkpoint.wait=94 checkpoint_threads=4 compact.free_space_target=63 debug.background_compact=0 debug.checkpoint_retention=7 debug.cursor_reposition=0 debug.eviction=0 debug.log_retention=0 debug.realloc_exact=0 debug.realloc_malloc=0 debug.slow_checkpoint=0 debug.table_logging=0 debug.update_restore_evict=0 disagg.internal_page_delta=1 disagg.leaf_page_delta=1 disagg.multi=0 disagg.multi_validation=0 disagg.enabled=1 disagg.layered=1 disagg.mode=switch disagg.page_log=palite disagg.key_provider=0 disagg.page_log.verbose=0 disagg.drain_threads=13 disagg.preserve=0 disk.data_extend=0 disk.encryption=none disk.mmap=1 disk.mmap_all=0 eviction.evict_use_softptr=1 file_manager.close_handle_minimum=14 file_manager.close_idle_time=33 file_manager.close_scan_interval=28 format.abort=0 format.independent_thread_rng=1 format.major_timeout=0 import=0 logging=0 logging.compression=none logging.file_max=106545 logging.prealloc=0 logging.remove=1 obsolete_cleanup.method=off obsolete_cleanup.wait=2566 ops.alter=0 ops.compaction=0 ops.hs_cursor=0 ops.bound_cursor=0 ops.prepare=1 ops.reserve=15 ops.random_cursor=0 ops.salvage=0 ops.throttle=0 ops.throttle.sleep_us=4690 ops.verify=1 prefetch=0 prefetch.default=0 precise_checkpoint=1 preserve_prepared=1 quiet=1 random.data_seed=14315905 random.extra_seed=5450710 rollback_to_stable_threads=4 runs.in_memory=0 runs.ops=0 runs.predictable_replay=0 runs.source=layered runs.tables=3 runs.threads=8 runs.timer=11 runs.type=row-store runs.verify_failure_dump=0 statistics.mode=fast statistics_log.sources=off stress.aggressive_stash_free=1 stress.aggressive_sweep=0 stress.checkpoint=0 stress.checkpoint_evict_page=0 stress.checkpoint_prepare=0 stress.compact_slow=0 stress.evict_reposition=0 stress.failpoint_eviction_split=1 stress.failpoint_hs_delete_key_from_ts=1 stress.failpoint_rec_before_wrapup=0 stress.hs_checkpoint_delay=0 stress.hs_search=0 stress.hs_sweep=0 stress.prefetch_delay=0 stress.prepare_resolution_1=0 stress.sleep_before_read_overflow_onpage=0 stress.split_1=0 stress.split_2=0 stress.split_3=0 stress.split_4=0 stress.split_5=0 stress.split_6=0 stress.split_7=0 stress.split_8=0 tiered_storage.flush_frequency=0 tiered_storage.storage_source=off transaction.implicit=0 transaction.operation_timeout_ms=2000 transaction.timestamps=1 wiredtiger.config=off wiredtiger.rwlock=1 wiredtiger.leak_memory=0 ############################################ # TABLE PARAMETERS: table 1 ############################################ table1.btree.compression=none table1.btree.dictionary=0 table1.btree.internal_key_truncation=1 table1.btree.internal_page_max=13 table1.btree.key_max=88 table1.btree.key_min=26 table1.btree.leaf_page_max=14 table1.btree.memory_page_max=6 table1.btree.prefix_len=0 table1.btree.prefix_compression=1 table1.btree.prefix_compression_min=7 table1.btree.split_pct=65 table1.btree.value_max=2060 table1.btree.value_min=17 table1.disk.checksum=on table1.disk.firstfit=0 table1.ops.pareto=0 table1.ops.pareto.skew=30 table1.ops.pct.delete=4 table1.ops.pct.insert=24 table1.ops.pct.modify=42 table1.ops.pct.read=2 table1.ops.pct.write=28 table1.ops.truncate=1 table1.runs.mirror=0 table1.runs.rows=3205 ############################################ # TABLE PARAMETERS: table 2 ############################################ table2.btree.compression=none table2.btree.dictionary=0 table2.btree.internal_key_truncation=1 table2.btree.internal_page_max=10 table2.btree.key_max=42 table2.btree.key_min=27 table2.btree.leaf_page_max=16 table2.btree.memory_page_max=2 table2.btree.prefix_len=0 table2.btree.prefix_compression=1 table2.btree.prefix_compression_min=6 table2.btree.split_pct=60 table2.btree.value_max=2714 table2.btree.value_min=15 table2.disk.checksum=on table2.disk.firstfit=0 table2.ops.pareto=1 table2.ops.pareto.skew=1 table2.ops.pct.delete=0 table2.ops.pct.insert=0 table2.ops.pct.modify=0 table2.ops.pct.read=0 table2.ops.pct.write=100 table2.ops.truncate=1 table2.runs.mirror=0 table2.runs.rows=742570 ############################################ # TABLE PARAMETERS: table 3 ############################################ table3.btree.compression=none table3.btree.dictionary=0 table3.btree.internal_key_truncation=1 table3.btree.internal_page_max=9 table3.btree.key_max=55 table3.btree.key_min=17 table3.btree.leaf_page_max=16 table3.btree.memory_page_max=3 table3.btree.prefix_len=0 table3.btree.prefix_compression=1 table3.btree.prefix_compression_min=8 table3.btree.split_pct=75 table3.btree.value_max=1656 table3.btree.value_min=4 table3.disk.checksum=uncompressed table3.disk.firstfit=0 table3.ops.pareto=0 table3.ops.pareto.skew=36 table3.ops.pct.delete=2 table3.ops.pct.insert=16 table3.ops.pct.modify=31 table3.ops.pct.read=14 table3.ops.pct.write=37 table3.ops.truncate=1 table3.runs.mirror=0 table3.runs.rows=740379 Command 'shell.exec' in function 'format test disagg' (step 2 of 2) failed: shell script encountered problem: exit code 1. Finished command 'shell.exec' in function 'format test disagg' (step 2 of 2) in 5m34.889099907s. Running task commands failed: running command: command failed: shell script encountered problem: exit code 1 Finished running task commands in 5m52.897599625s. Task completed - FAILURE. Running post-task commands.
format-stress-test-disagg-switch-1 task_log
Logs:
#0 0x0000ffff93ac4454 in __pthread_kill_implementation () from /lib64/libc.so.6 #0 0x0000ffff93ac4454 in __pthread_kill_implementation () from /lib64/libc.so.6 #1 0x0000ffff93a7b320 [PAC] in raise () from /lib64/libc.so.6 #2 0x0000ffff93a62224 [PAC] in abort () from /lib64/libc.so.6 #3 0x0000ffff94665ccc [PAC] in __wt_abort (session=0xffff93213ee0) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/os_common/os_abort.c:32 #4 0x0000ffff943c5dd4 in __curhs_insert (cursor=0x51600011a980) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/cursor/cur_hs.c:1068 #5 0x0000ffff947f419c in __rec_hs_delete_reinsert_from_pos (session=0xffff93213ee0, hs_cursor=0x5160001f6880, btree_id=41, key=0x504001d7c290, ts=1, reinsert=true, no_ts_tombstone=false, error_on_ts_ordering=false, counter=0xffff8610e0c0) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/reconcile/rec_hs.c:224 #6 0x0000ffff947f04d4 in __rec_hs_insert_record (session=0xffff93213ee0, cursor=0x5160001f6880, btree=0x5180003d6880, ref=0x50e0002e2920, key=0x504001d7c290, type=3 '\003', hs_value=0x5040028b6350, tw=0xffff86202620, error_on_ts_ordering=false) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/reconcile/rec_hs.c:533 #7 0x0000ffff947e901c in __wti_rec_hs_insert_updates (session=0xffff93213ee0, r=0x51e000a4e880, multi=0x51900038b880) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/reconcile/rec_hs.c:1137 #8 0x0000ffff947d72d8 in __rec_hs_wrapup (session=0xffff93213ee0, r=0x51e000a4e880) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/reconcile/rec_write.c:3334 #9 0x0000ffff947a9484 in __rec_write_wrapup (session=0xffff93213ee0, r=0x51e000a4e880) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/reconcile/rec_write.c:2893 #10 0x0000ffff9479638c in __reconcile (session=0xffff93213ee0, ref=0x50e0002e2920, salvage=0x0, flags=672, page_lockedp=0xffff85f58020) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/reconcile/rec_write.c:395 #11 0x0000ffff9478fa54 in __wt_reconcile (session=0xffff93213ee0, ref=0x50e0002e2920, salvage=0x0, flags=672) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/reconcile/rec_write.c:127 #12 0x0000ffff945500bc in __evict_reconcile (session=0xffff93213ee0, ref=0x50e0002e2920, evict_flags=0) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/evict/evict_page.c:1277 #13 0x0000ffff9454b7f0 in __wt_evict (session=0xffff93213ee0, ref=0x50e0002e2920, previous_state=3 '\003', flags=0) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/evict/evict_page.c:443 #14 0x0000ffff94522618 in __evict_page (session=0xffff93213ee0, is_server=false) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/evict/evict_lru.c:3113 #15 0x0000ffff94527d84 in __evict_lru_pages (session=0xffff93213ee0, is_server=false) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/evict/evict_lru.c:1444 #16 0x0000ffff9451a300 in __evict_thread_run (session=0xffff93213ee0, thread=0x507000c81010) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/evict/evict_lru.c:381 #17 0x0000ffff949fab50 in __thread_run (arg=0x507000c81010) at /data/mci/133ed9e5ccf32ca633a2f8e223a9b9d3/wiredtiger/src/support/thread_group.c:32 #18 0x0000ffff951056e0 in asan_thread_start () at /data/mci/f49c90f0be13703fc74cd438c017c2d0/toolchain-builder/tmp/build-llvm-v5.sh-WmZ/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:239 #19 0x0000ffff93ac2834 in start_thread () from /lib64/libc.so.6 #20 0x0000ffff93a66e5c [PAC] in thread_start () from /lib64/libc.so.6