-
Type:
Build Failure
-
Resolution: Unresolved
-
Priority:
Major - P3
-
None
-
Affects Version/s: None
-
Component/s: Disagg CI-blocker
-
None
-
Storage Engines - Foundations
-
188.458
-
None
-
None
-
1
format-stress-test-disagg-multi-1 on amazon2023-disagg-stress
Host: i-083823da638b701fa
Project: wiredtiger-disagg
Commit: 84c1ab92
Please refer to BF(G) Playbook for instructions on handling BF and BFG tickets as well as Auto-Resolution Rules
Task Logs:
format-stress-test-disagg-multi-1 task_log
Logs:
Task logger initialized (agent version '2026-05-01a' from Evergreen build revision '506cfa5dd480356b3052925f267bd0aabe7d8ec5').
Starting task 'wiredtiger_disagg_amazon2023_disagg_stress_format_stress_test_disagg_multi_1_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53', execution 0.
Running pre-task commands.
Running command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre'.
Finished command 'shell.exec' in function 'cleanup' (step 1 of 2) in block 'pre' in 2.391802ms.
Running command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre'.
Finished command 'expansions.update' in function 'setup environment' (step 2 of 2) in block 'pre' in 55.407µs.
Finished running pre-task commands in 2.629373ms.
Running task commands.
Running command 's3.get' in function 'fetch artifacts' (step 1 of 2).
Fetching remote file 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/compile_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53.tgz' from S3 bucket 'build_external' (attempt 1 of 5).
Finished command 's3.get' in function 'fetch artifacts' (step 1 of 2) in 9.135768548s.
Running command 'shell.exec' in function 'format test disagg' (step 2 of 2).
export WT_TOPDIR=$(git rev-parse --show-toplevel)
export WT_BUILDDIR=$WT_TOPDIR/cmake_build
if [ "$OS" = "Windows_NT" ]; then
export PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH
else
export PATH=/opt/mongodbtoolchain/v5/bin:$PATH
export LD_LIBRARY_PATH=$WT_BUILDDIR
fi
# Create the common sanitizer options and export the specific sanitizer environment
# variables.
COMMON_SAN_OPTIONS="abort_on_error=1:disable_coredump=0"
if [[ "" =~ ASan ]]; then
export ASAN_OPTIONS="$COMMON_SAN_OPTIONS:unmap_shadow_on_exit=1"
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(dirname $(/opt/mongodbtoolchain/v5/bin/clang -print-file-name=libclang_rt.asan.so))"
export TESTUTIL_BYPASS_ASAN=1
elif [[ "" =~ MSan ]]; then
export MSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
export TESTUTIL_MSAN=1
elif [[ "" =~ TSan ]]; then
export TSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
export TESTUTIL_TSAN=1
elif [[ "" =~ UBSan ]]; then
export UBSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_stacktrace=1"
export TESTUTIL_UBSAN=1
fi
if [[ 1 -eq 1 ]]; then
# The preloaded library should be downloaded/built during "configure wiredtiger" step.
# DO NOT MOVE:This variable must be set in the same function(shell) that runs a test,
# and as close as possible to the test execution, since it affects all the binaries.
export LD_PRELOAD=$WT_TOPDIR/TCMALLOC_LIB/libtcmalloc.so
if [ ! -f "$LD_PRELOAD" ]; then
echo "Error: TCMalloc support was requested, but the library doesn't exist: $LD_PRELOAD"
exit 1
fi
fi
# Set extensions to load
EXT="extensions=["
EXT="$EXT ext/compressors/snappy/libwiredtiger_snappy.so,"
EXT="$EXT ext/collators/reverse/libwiredtiger_reverse_collator.so,"
EXT="$EXT ext/encryptors/rotn/libwiredtiger_rotn.so,"
EXT="$EXT ext/page_log/palite/libwiredtiger_palite.so,"
EXT="$EXT ]"
export EXT
Dump Environment
eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
BASH_FUNC_which%%=() { ( alias;
CI=true
EVR_AGENT_PID=3569
EVR_TASK_ID=wiredtiger_disagg_amazon2023_disagg_stress_format_stress_test_disagg_multi_1_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53
EXT=extensions=[ ext/compressors/snappy/libwiredtiger_snappy.so, ext/collators/reverse/libwiredtiger_reverse_collator.so, ext/encryptors/rotn/libwiredtiger_rotn.so, ext/page_log/palite/libwiredtiger_palite.so, ]
GOCACHE=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wiredtiger/cmake_build/test/format/.gocache
GOTRACEBACK=none
HISTCONTROL=ignoredups
HISTSIZE=1000
HOME=/home/ec2-user
HOSTNAME=ip-10-122-50-92.ec2.internal
INVOCATION_ID=4b77f3f9865f4df28daafa33435508ed
JASPER_ID=a3f63898-8fc3-470e-9211-adc506336f5c
JASPER_MANAGER=6be1a122-ba73-4304-a43a-afdee19aa570
JOURNAL_STREAM=8:21001
LANG=C.UTF-8
LC_ALL=C
LD_LIBRARY_PATH=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wiredtiger/cmake_build
LD_PRELOAD=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wiredtiger/TCMALLOC_LIB/libtcmalloc.so
LESSOPEN=||/usr/bin/lesspipe.sh %s
LOGNAME=ec2-user
MAIL=/var/spool/mail/ec2-user
PATH=/opt/mongodbtoolchain/v5/bin:/usr/sbin:/usr/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/node/bin:/opt/node/bin:/opt/node/bin
PWD=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wiredtiger/cmake_build/test/format
SHELL=/bin/bash
SHLVL=1
SYSTEMD_COLORS=false
SYSTEMD_EXEC_PID=3363
S_COLORS=auto
TEMP=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/tmp
TMP=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/tmp
TMPDIR=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/tmp
USER=ec2-user
WT_BUILDDIR=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wiredtiger/cmake_build
WT_TOPDIR=/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wiredtiger
_=/usr/bin/printenv
which_declare=declare -f
}
echo "Dump Environment"
printenv | sort
# Fail, show the configuration file.
fail() {
echo "======= FAILURE =========="
[ -f RUNDIR/CONFIG ] && cat RUNDIR/CONFIG
exit 1
}
for i in $(seq 5); do
echo Iteration $i/5
rm -rf RUNDIR
./t -c ../../../test/format/CONFIG.disagg disagg.multi=1 runs.predictable_replay=1 runs.ops=500000:2000000 ops.pct.delete=0 || fail
Iteration 1/5
===> Output will be written to RUNDIR/leader.out
(If you want to watch live, run: tail -f RUNDIR/leader.out)
===> Output will be written to RUNDIR/follower/follower.out
(If you want to watch live, run: tail -f RUNDIR/follower/follower.out)
Iteration 2/5
===> Output will be written to RUNDIR/leader.out
(If you want to watch live, run: tail -f RUNDIR/leader.out)
===> Output will be written to RUNDIR/follower/follower.out
(If you want to watch live, run: tail -f RUNDIR/follower/follower.out)
Iteration 3/5
===> Output will be written to RUNDIR/leader.out
(If you want to watch live, run: tail -f RUNDIR/leader.out)
===> Output will be written to RUNDIR/follower/follower.out
(If you want to watch live, run: tail -f RUNDIR/follower/follower.out)
Iteration 4/5
===> Output will be written to RUNDIR/leader.out
(If you want to watch live, run: tail -f RUNDIR/leader.out)
===> Output will be written to RUNDIR/follower/follower.out
(If you want to watch live, run: tail -f RUNDIR/follower/follower.out)
Iteration 5/5
===> Output will be written to RUNDIR/leader.out
(If you want to watch live, run: tail -f RUNDIR/leader.out)
===> Output will be written to RUNDIR/follower/follower.out
(If you want to watch live, run: tail -f RUNDIR/follower/follower.out)
if [ false = true ]; then
./t -R $format_args || fail
fi
done
bash: line 65: 27643 Aborted (core dumped) ./t -c ../../../test/format/CONFIG.disagg disagg.multi=1 runs.predictable_replay=1 runs.ops=500000:2000000 ops.pct.delete=0
======= FAILURE ==========
############################################
# RUN PARAMETERS: V3
############################################
assert.read_timestamp=0
background_compact=0
background_compact.free_space_target=77
backup=0
backup.incremental=off
backup.incr_granularity=9140
backup.live_restore=0
backup.live_restore_read_size=16384
backup.live_restore_threads=0
block_cache=0
block_cache.cache_on_checkpoint=1
block_cache.cache_on_writes=0
block_cache.size=2
btree.reverse=0
cache=3072
cache.evict_max=1
cache.eviction_dirty_target=0
cache.eviction_dirty_trigger=0
cache.eviction_updates_target=0
cache.eviction_updates_trigger=0
cache.minimum=0
cache.maximum=0
checkpoint=on
checkpoint.log_size=192
checkpoint.wait=85
checkpoint_threads=3
compact.free_space_target=1
debug.background_compact=0
debug.checkpoint_retention=7
debug.cursor_reposition=0
debug.eviction=0
debug.log_retention=3
debug.realloc_exact=0
debug.realloc_malloc=0
debug.slow_checkpoint=0
debug.table_logging=0
debug.update_restore_evict=0
disagg.internal_page_delta=1
disagg.leaf_page_delta=1
disagg.multi=1
disagg.multi_validation=0
disagg.enabled=1
disagg.layered=1
disagg.mode=leader
disagg.page_log=palite
disagg.key_provider=0
disagg.page_log.verbose=0
disagg.drain_threads=10
disagg.preserve=0
disk.data_extend=0
disk.encryption=none
disk.mmap=1
disk.mmap_all=0
eviction.evict_use_softptr=1
file_manager.close_handle_minimum=28
file_manager.close_idle_time=7
file_manager.close_scan_interval=29
format.abort=0
format.independent_thread_rng=1
format.major_timeout=0
import=0
logging=0
logging.compression=none
logging.file_max=498326
logging.prealloc=0
logging.remove=0
obsolete_cleanup.method=off
obsolete_cleanup.wait=1370
ops.alter=0
ops.compaction=0
ops.hs_cursor=0
ops.pct.delete=0
ops.bound_cursor=0
ops.prepare=1
ops.reserve=9
ops.random_cursor=0
ops.salvage=0
ops.throttle=0
ops.throttle.sleep_us=844821
ops.verify=1
prefetch=1
prefetch.default=0
precise_checkpoint=1
preserve_prepared=1
quiet=1
random.data_seed=7297827
random.extra_seed=11076768
rollback_to_stable_threads=9
runs.in_memory=0
runs.ops=1400000
runs.predictable_replay=1
runs.source=layered
runs.tables=3
runs.threads=1
runs.timer=360
runs.type=row-store
runs.verify_failure_dump=0
statistics.mode=fast
statistics_log.sources=off
stress.aggressive_stash_free=0
stress.aggressive_sweep=0
stress.checkpoint=0
stress.checkpoint_evict_page=0
stress.checkpoint_prepare=0
stress.compact_slow=0
stress.evict_reposition=0
stress.failpoint_eviction_split=0
stress.failpoint_hs_delete_key_from_ts=1
stress.failpoint_rec_before_wrapup=0
stress.hs_checkpoint_delay=0
stress.hs_search=0
stress.hs_sweep=0
stress.prefetch_delay=0
stress.prepare_resolution_1=0
stress.sleep_before_read_overflow_onpage=0
stress.split_1=0
stress.split_2=1
stress.split_3=0
stress.split_4=0
stress.split_5=0
stress.split_6=0
stress.split_7=0
stress.split_8=0
tiered_storage.flush_frequency=0
tiered_storage.storage_source=off
transaction.implicit=0
transactio
format-stress-test-disagg-multi-1 task_log
Logs:
#0 0x0000ffff890c4454 in __pthread_kill_implementation () from /lib64/libc.so.6 #0 0x0000ffff890c4454 in __pthread_kill_implementation () from /lib64/libc.so.6 #1 0x0000ffff8907b320 [PAC] in raise () from /lib64/libc.so.6 #2 0x0000ffff89062224 [PAC] in abort () from /lib64/libc.so.6 #3 0x0000ffff894468dc [PAC] in __wt_abort (session=session@entry=0x1356ffc915a0) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/os_common/os_abort.c:32 #4 0x0000ffff8952f7ac in __txn_assert_after_reads (session=session@entry=0x1356ffc915a0, op=op@entry=0xffff8959be38 "prepare", ts=ts@entry=263596) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/txn/txn_timestamp.c:556 #5 0x0000ffff8952fe20 in __txn_set_prepare_timestamp (session=session@entry=0x1356ffc915a0, prepare_ts=prepare_ts@entry=263596) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/txn/txn_timestamp.c:824 #6 0x0000ffff89532900 in __wt_txn_set_timestamp_uint (session=session@entry=0x1356ffc915a0, which=which@entry=WT_TS_TXN_TYPE_PREPARE, ts=ts@entry=263596) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/txn/txn_timestamp.c:1185 #7 0x0000ffff894bc4c0 in __session_timestamp_transaction_uint (wt_session=0x1356ffc915a0, which=WT_TS_TXN_TYPE_PREPARE, ts=263596) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/session/session_api.c:2139 #8 0x0000000000415b78 in prepare_transaction (tinfo=tinfo@entry=0x1356da602000) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/test/format/ops.c:703 #9 0x0000000000416e58 in ops (arg=0x1356da602000) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/test/format/ops.c:1409 #10 0x0000ffff890c2834 in start_thread () from /lib64/libc.so.6 #11 0x0000ffff89066e5c [PAC] in thread_start () from /lib64/libc.so.6 #0 0x0000ffff890c4454 in __pthread_kill_implementation () from /lib64/libc.so.6 #0 0x0000ffff890c4454 in __pthread_kill_implementation () from /lib64/libc.so.6 #1 0x0000ffff8907b320 [PAC] in raise () from /lib64/libc.so.6 #2 0x0000ffff89062224 [PAC] in abort () from /lib64/libc.so.6 #3 0x0000ffff894468dc [PAC] in __wt_abort (session=session@entry=0x1356ffc915a0) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/os_common/os_abort.c:32 #4 0x0000ffff8952f7ac in __txn_assert_after_reads (session=session@entry=0x1356ffc915a0, op=op@entry=0xffff8959be38 "prepare", ts=ts@entry=187056) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/txn/txn_timestamp.c:556 #5 0x0000ffff8952fe20 in __txn_set_prepare_timestamp (session=session@entry=0x1356ffc915a0, prepare_ts=prepare_ts@entry=187056) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/txn/txn_timestamp.c:824 #6 0x0000ffff89532900 in __wt_txn_set_timestamp_uint (session=session@entry=0x1356ffc915a0, which=which@entry=WT_TS_TXN_TYPE_PREPARE, ts=ts@entry=187056) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/txn/txn_timestamp.c:1185 #7 0x0000ffff894bc4c0 in __session_timestamp_transaction_uint (wt_session=0x1356ffc915a0, which=WT_TS_TXN_TYPE_PREPARE, ts=187056) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/src/session/session_api.c:2139 #8 0x0000000000415b78 in prepare_transaction (tinfo=tinfo@entry=0x1356d04cca80) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/test/format/ops.c:703 #9 0x0000000000416e58 in ops (arg=0x1356d04cca80) at /data/mci/42f54761123900557c33cb879e4d0e48/wiredtiger/test/format/ops.c:1409 #10 0x0000ffff890c2834 in start_thread () from /lib64/libc.so.6 #11 0x0000ffff89066e5c [PAC] in thread_start () from /lib64/libc.so.6
format-stress-test-disagg-multi-1 task_log
Logs:
Finished command 'shell.exec' in function 'dump stacktraces' (step 1 of 9) in block 'post' in 4.118771388s.
Running command 's3.put' in function 'upload stacktraces' (step 2 of 9) in block 'post'.
Putting files matching filter '[wiredtiger/cmake_build/*stacktrace.txt]' into path 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/' in S3 bucket 'build_external'.
Performing S3 put to file 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/' in bucket 'build_external' (attempt 1 of 5).
Finished command 's3.put' in function 'upload stacktraces' (step 2 of 9) in block 'post' in 211.016439ms.
Running command 's3.put' in function 'upload test/format config' (step 3 of 9) in block 'post'.
Putting files matching filter '[wiredtiger/cmake_build/test/format/RUNDIR*/CONFIG]' into path 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/' in S3 bucket 'build_external'.
Performing S3 put to file 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/' in bucket 'build_external' (attempt 1 of 5).
Finished command 's3.put' in function 'upload test/format config' (step 3 of 9) in block 'post' in 139.470956ms.
Running command 's3.put' in function 'upload test/model workloads' (step 4 of 9) in block 'post'.
Putting files matching filter '[wiredtiger/cmake_build/test/model/tools/WT_TEST*/*.workload]' into path 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/' in S3 bucket 'build_external'.
Performing S3 put to file 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/' in bucket 'build_external' (attempt 1 of 5).
File filter 'wiredtiger/cmake_build/test/model/tools/WT_TEST*/*.workload' matched no files.
Finished command 's3.put' in function 'upload test/model workloads' (step 4 of 9) in block 'post' in 42.27265ms.
Running command 'shell.exec' in function 'dump stderr/stdout' (step 5 of 9) in block 'post'.
if [ -d "WT_TEST" ]; then
# Dump stderr/stdout contents generated by the C libraries onto console for Python tests
find "WT_TEST" -name "std*.txt" ! -empty -exec bash -c "echo 'Contents from {}:'; cat '{}'" \;
fi
Finished command 'shell.exec' in function 'dump stderr/stdout' (step 5 of 9) in block 'post' in 1.379269ms.
Running command 'shell.exec' in function 'upload stat files' (step 6.1 of 9) in block 'post'.
mkdir stat_files
python3 ../test/evergreen/collect_stat_files.py -d stat_files
Finished command 'shell.exec' in function 'upload stat files' (step 6.1 of 9) in block 'post' in 33.796067ms.
Running command 'archive.targz_pack' in function 'upload stat files' (step 6.2 of 9) in block 'post'.
Beginning to build archive.
[message='successfully created archive' num_files='4' target='/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wiredtiger/cmake_build/stat_files.tar.gz']
Finished command 'archive.targz_pack' in function 'upload stat files' (step 6.2 of 9) in block 'post' in 14.305374ms.
Running command 's3.put' in function 'upload stat files' (step 6.3 of 9) in block 'post'.
Putting local file 'wiredtiger/cmake_build/stat_files.tar.gz' into path 'build_external/wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/stat_files.tar.gz' (https://build_external.s3.amazonaws.com/wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/stat_files.tar.gz).
Performing S3 put to file 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53/stat_files.tar.gz' in bucket 'build_external' (attempt 1 of 5).
Finished command 's3.put' in function 'upload stat files' (step 6.3 of 9) in block 'post' in 136.811219ms.
Running command 'archive.targz_pack' in function 'upload artifact' (step 7.1 of 9) in block 'post'.
Beginning to build archive.
[message='successfully created archive' num_files='7003' target='/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wiredtiger.tgz']
Finished command 'archive.targz_pack' in function 'upload artifact' (step 7.1 of 9) in block 'post' in 1.943129275s.
Running command 's3.put' in function 'upload artifact' (step 7.2 of 9) in block 'post'.
Putting local file 'wiredtiger.tgz' into path 'build_external/wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53-0.tgz' (https://build_external.s3.amazonaws.com/wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53-0.tgz).
Performing S3 put to file 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/artifacts/format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53-0.tgz' in bucket 'build_external' (attempt 1 of 5).
Finished command 's3.put' in function 'upload artifact' (step 7.2 of 9) in block 'post' in 1.411302067s.
Running command 'archive.targz_pack' in function 'save wt hang analyzer core/debugger files' (step 8.1 of 9) in block 'post'.
Beginning to build archive.
No files were archived.
Finished command 'archive.targz_pack' in function 'save wt hang analyzer core/debugger files' (step 8.1 of 9) in block 'post' in 331.576µs.
Running command 's3.put' in function 'save wt hang analyzer core/debugger files' (step 8.2 of 9) in block 'post'.
Putting local file 'wt-hang-analyzer.tgz' into path 'build_external/wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/wt_hang_analyzer/wt-hang-analyzer_format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53.tgz' (https://build_external.s3.amazonaws.com/wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/wt_hang_analyzer/wt-hang-analyzer_format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53.tgz).
Performing S3 put to file 'wiredtiger/amazon2023-disagg-stress/84c1ab9229f36d61fc730b7d2e217c90d7315724/wt_hang_analyzer/wt-hang-analyzer_format-stress-test-disagg-multi-1_wiredtiger_disagg_amazon2023_disagg_stress_84c1ab9229f36d61fc730b7d2e217c90d7315724_26_05_07_23_17_53.tgz' in bucket 'build_external' (attempt 1 of 5).
File '/data/mci/2e4016cbf8a6feb54ba629bd5df61b44/wt-hang-analyzer.tgz' not found and skip missing is true, exiting without error.
Finished command 's3.put' in function 'save wt hang analyzer core/debugger files' (step 8.2 of 9) in block 'post' in 47.288754ms.
Running command 'shell.exec' in function 'cleanup' (step 9 of 9) in block 'post'.
Finished command 'shell.exec' in function 'cleanup' (step 9 of 9) in block 'post' in 684.492422ms.
Finished running post-task commands in 8.785797831s.
reading test log spec; falling back to default spec: open /data/mci/2e4016cbf8a6feb54ba629bd5df61b44/build/TestLogs/log_spec.yaml: no such file or directory