Task logger initialized (agent version '2026-07-22' from Evergreen build revision 'df31613557cd78fda1a8b957c69085c8f2b79f47').
Starting task 'wiredtiger_disagg_amazon2023_disagg_asan_stress_schema_disagg_abort_smoke_test_disagg_1_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38', execution 0.
Running pre-task commands.
Running command 'shell.exec' in function 'cleanup' (step 1 of 3) in block 'pre'.
Finished command 'shell.exec' in function 'cleanup' (step 1 of 3) in block 'pre' in 2.592865ms.
Running command 'expansions.update' in function 'setup environment' (step 2 of 3) in block 'pre'.
Finished command 'expansions.update' in function 'setup environment' (step 2 of 3) in block 'pre' in 40.721µs.
Running command 'shell.exec' in function 'pin mongo toolchain' (step 3 of 3) in block 'pre'.
mongodbtoolchain 93d85cc1a00ca1d53fcc7d4ca790f19a4e5dd542 not installed, downloading...
Autodetected Amazon Linux 2023 (OS_ID: amazon2023) on aarch64
Autodetected DOWNLOAD_URL: https://mongodbtoolchain.build.10gen.cc/toolchain/amazon2023-arm64/x86_64/latest
Overriding REVISION_ID, will download toolchain for 93d85cc1a00ca1d53fcc7d4ca790f19a4e5dd542
Ensuring /opt/mongodbtoolchain/revisions exists and has correct ownership...
Downloading toolchain to /data/mci/ba300d3d466c2a95109fe0c8f79cd817/tmp/tmp.mBp43Gvfkb/mongodbtoolchain.tar.gz from https://mongodbtoolchain.build.10gen.cc/toolchain/amazon2023-arm64/x86_64/mongodbtoolchain-93d85cc1a00ca1d53fcc7d4ca790f19a4e5dd542.tar.gz
Getting toolchain revision from archive...
Checking downloaded archive integrity...
Extracting toolchain revision 93d85cc1a00ca1d53fcc7d4ca790f19a4e5dd542 from archive...
Running post install script...
Existing v2 found, uninstalling...
Existing v3 found, uninstalling...
Existing v4 found, uninstalling...
Existing v5 found, uninstalling...
Installing v2...
Installing v3...
Installing v4...
Installing v5...
Cleaning up...
Finished command 'shell.exec' in function 'pin mongo toolchain' (step 3 of 3) in block 'pre' in 3m8.539531921s.
Finished running pre-task commands in 3m8.542469383s.
Running task commands.
Running command 's3.get' in function 'fetch artifacts' (step 1 of 2).
Fetching remote file 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/compile_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38.tgz' from S3 bucket 'build_external' (attempt 1 of 5).
Finished command 's3.get' in function 'fetch artifacts' (step 1 of 2) in 7.693062893s.
Running command 'shell.exec' (step 2 of 2).
export WT_TOPDIR=$(git rev-parse --show-toplevel)
export WT_BUILDDIR=$WT_TOPDIR/cmake_build
if [ "$OS" = "Windows_NT" ]; then
  export PATH=/cygdrive/c/python/Python311:/cygdrive/c/python/Python311/Scripts:$PATH
else
  export PATH=/opt/mongodbtoolchain/v5/bin:$PATH
  export LD_LIBRARY_PATH=$WT_BUILDDIR
fi
# Create the common sanitizer options and export the specific sanitizer environment
# variables.
COMMON_SAN_OPTIONS="abort_on_error=1:disable_coredump=0"
if [[ "-DCMAKE_BUILD_TYPE=ASan" =~ ASan ]]; then
  export ASAN_OPTIONS="$COMMON_SAN_OPTIONS:unmap_shadow_on_exit=1"
  asan_lib=$(clang -print-file-name=libclang_rt.asan.so)
  [ -f "$asan_lib" ] || { echo "ERROR: libclang_rt.asan.so not found by clang"; exit 1; }
  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(dirname "$asan_lib")"
  export TESTUTIL_BYPASS_ASAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ MSan ]]; then
  export MSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
  export TESTUTIL_MSAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ TSan ]]; then
  export TSAN_OPTIONS="$COMMON_SAN_OPTIONS:verbosity=3"
  export TESTUTIL_TSAN=1
elif [[ "-DCMAKE_BUILD_TYPE=ASan" =~ UBSan ]]; then
  export UBSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_stacktrace=1"
  export TESTUTIL_UBSAN=1
fi
if [[ 0 -eq 1 ]]; then
  # The preloaded library should be downloaded/built during "configure wiredtiger" step.
  # DO NOT MOVE:This variable must be set in the same function(shell) that runs a test,
  # and as close as possible to the test execution, since it affects all the binaries.
  export LD_PRELOAD=$WT_TOPDIR/TCMALLOC_LIB/libtcmalloc.so
  if [ ! -f "$LD_PRELOAD" ]; then
    echo "Error: TCMalloc support was requested, but the library doesn't exist: $LD_PRELOAD"
    exit 1
  fi
fi
# Set extensions to load
EXT="extensions=["
EXT="$EXT ext/compressors/snappy/libwiredtiger_snappy.so,"
EXT="$EXT ext/collators/reverse/libwiredtiger_reverse_collator.so,"
EXT="$EXT ext/encryptors/rotn/libwiredtiger_rotn.so,"
EXT="$EXT ext/page_log/palite/libwiredtiger_palite.so,"
EXT="$EXT ]"
export EXT
export LSAN_OPTIONS="$COMMON_SAN_OPTIONS:print_suppressions=0:suppressions=$(git rev-parse --show-toplevel)/test/evergreen/asan_leaks.supp"
Dump Environment
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
ASAN_OPTIONS=abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
BASH_FUNC_which%%=() {  ( alias;
CI=true
EVR_AGENT_PID=3617
EVR_TASK_ID=wiredtiger_disagg_amazon2023_disagg_asan_stress_schema_disagg_abort_smoke_test_disagg_1_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38
EXT=extensions=[ ext/compressors/snappy/libwiredtiger_snappy.so, ext/collators/reverse/libwiredtiger_reverse_collator.so, ext/encryptors/rotn/libwiredtiger_rotn.so, ext/page_log/palite/libwiredtiger_palite.so, ]
GOCACHE=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/cmake_build/test/csuite/schema_disagg_abort/.gocache
GOTRACEBACK=none
HISTCONTROL=ignoredups
HISTSIZE=1000
HOME=/home/ec2-user
HOSTNAME=ip-10-128-26-163.ec2.internal
INVOCATION_ID=00964ce19fb743319f71afad50b64c2a
JASPER_ID=d6f75cd0-cd57-42db-bf70-84532ed2fb51
JASPER_MANAGER=e405b275-49b4-412b-8550-c3a10ff3dea7
JOURNAL_STREAM=8:24939
LANG=C.UTF-8
LC_ALL=C
LD_LIBRARY_PATH=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/cmake_build:/opt/mongodbtoolchain/revisions/93d85cc1a00ca1d53fcc7d4ca790f19a4e5dd542/stow/llvm-v5.1VC/lib/clang/19/lib/aarch64-mongodb-linux
LESSOPEN=||/usr/bin/lesspipe.sh %s
LOGNAME=ec2-user
LSAN_OPTIONS=abort_on_error=1:disable_coredump=0:print_suppressions=0:suppressions=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/test/evergreen/asan_leaks.supp
MAIL=/var/spool/mail/ec2-user
PATH=/opt/mongodbtoolchain/v5/bin:/usr/sbin:/usr/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/node/bin:/opt/node/bin:/opt/node/bin
PWD=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/cmake_build/test/csuite/schema_disagg_abort
SHELL=/bin/bash
SHLVL=1
SYSTEMD_COLORS=false
SYSTEMD_EXEC_PID=3388
S_COLORS=auto
TEMP=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/tmp
TESTUTIL_BYPASS_ASAN=1
TMP=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/tmp
TMPDIR=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/tmp
USER=ec2-user
WT_BUILDDIR=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/cmake_build
WT_TOPDIR=/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger
_=/usr/bin/printenv
which_declare=declare -f
}
echo "Dump Environment"
printenv | sort
# The full mode matrix in short runs: single node, lone follower, role switches,
# kills, and both two-node topologies. The two-node switch modes at the end are
# EXPECTED to fail until FIXME-WT-18068 and FIXME-WT-17746 land.
bash ./smoke.sh
Running test command: ./test_schema_disagg_abort -b /data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/cmake_build -r l -t 10 -T 2 -h WT_TEST.schema_disagg_abort.l
Parent: roles l; 2 schema threads; pool 32 slots; switch every 0s; kill leader@0 follower@0 lone@0; stop at 10s
CONFIG: test_schema_disagg_abort -r l -u 32 -T 2 -t 10 -PSD246114,E6666028
Running test command: /data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/cmake_build/test/csuite/schema_disagg_abort/test_schema_disagg_abort -r node -i 0 -A l -h WT_TEST.schema_disagg_abort.l -b /data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/cmake_build -T 2 -u 32 -PSD246114,E6666028
Node 0: starting as leader
Node 0: checkpoint 1 complete
Node 0: checkpoint 2 complete
Node 0: checkpoint 3 complete
Node 0: checkpoint 4 complete
Node 0: checkpoint 5 complete
Node 0: checkpoint 6 complete
Node 0: checkpoint 7 complete
Node 0: checkpoint 8 complete
Node 0: checkpoint 9 complete
Parent: directing graceful stop
Node 0: checkpoint 10 complete
Node 0: stopped gracefully as leader
Parent: Open node0 database, run recovery and verify content
[1785415111:071176][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: WiredTiger
[1785415111:071253][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: WiredTiger.basecfg
[1785415111:071273][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: WiredTiger.wt
[1785415111:071317][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: WiredTiger.turtle
[1785415111:071336][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: WiredTigerHS.wt
[1785415111:071353][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: schema_0_1_11.wt_ingest
[1785415111:071373][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: schema_0_1_12.wt_ingest
[1785415111:071393][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: schema_0_0_19.wt_ingest
[1785415111:071410][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: schema_0_1_2.wt_ingest
[1785415111:071426][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARNING]: Removing local file due to disagg mode: schema_0_0_28.wt_ingest
[1785415111:071442][13509:0xffff8ed71020], wiredtiger_open: [WT_VERB_METADATA][WARN

#0  0x0000ffffbc4c4454 in ?? ()
#0  0x0000ffffbc4c4454 in __pthread_kill_implementation () from /lib64/libc.so.6
#1  0x0000ffffbc47b320 [PAC] in raise () from /lib64/libc.so.6
#2  0x0000ffffbc462224 [PAC] in abort () from /lib64/libc.so.6
#3  0x0000ffffbd0cee70 [PAC] in __wt_abort (session=0x0) at /data/mci/0cfc8876461ecf2210da272566318de1/wiredtiger/src/os_common/os_abort.c:32
#4  0x0000aaaabd4ed418 in ?? ()
#5  0x0000fffff17544e0 in ?? ()

Finished command 'shell.exec' in function 'dump stacktraces' (step 1 of 9) in block 'post' in 2.225713924s.
Running command 's3.put' in function 'upload stacktraces' (step 2 of 9) in block 'post'.
Putting files matching filter '[wiredtiger/cmake_build/*stacktrace.txt]' into path 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/' in S3 bucket 'build_external'.
Performing S3 put to file 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/' in bucket 'build_external' (attempt 1 of 5).
Finished command 's3.put' in function 'upload stacktraces' (step 2 of 9) in block 'post' in 235.92285ms.
Running command 's3.put' in function 'upload test/format config' (step 3 of 9) in block 'post'.
Putting files matching filter '[wiredtiger/cmake_build/test/format/RUNDIR*/CONFIG]' into path 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/' in S3 bucket 'build_external'.
Performing S3 put to file 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/' in bucket 'build_external' (attempt 1 of 5).
File filter 'wiredtiger/cmake_build/test/format/RUNDIR*/CONFIG' matched no files.
Finished command 's3.put' in function 'upload test/format config' (step 3 of 9) in block 'post' in 53.070757ms.
Running command 's3.put' in function 'upload test/model workloads' (step 4 of 9) in block 'post'.
Putting files matching filter '[wiredtiger/cmake_build/test/model/tools/WT_TEST*/*.workload]' into path 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/' in S3 bucket 'build_external'.
Performing S3 put to file 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/' in bucket 'build_external' (attempt 1 of 5).
File filter 'wiredtiger/cmake_build/test/model/tools/WT_TEST*/*.workload' matched no files.
Finished command 's3.put' in function 'upload test/model workloads' (step 4 of 9) in block 'post' in 46.427778ms.
Running command 'shell.exec' in function 'dump stderr/stdout' (step 5 of 9) in block 'post'.
if [ -d "WT_TEST" ]; then
  # Dump stderr/stdout contents generated by the C libraries onto console for Python tests
  find "WT_TEST" -name "std*.txt" ! -empty -exec bash -c "echo 'Contents from {}:'; cat '{}'" \;
fi
Finished command 'shell.exec' in function 'dump stderr/stdout' (step 5 of 9) in block 'post' in 1.377116ms.
Running command 'shell.exec' in function 'upload stat files' (step 6.1 of 9) in block 'post'.
mkdir stat_files
python3 ../test/evergreen/collect_stat_files.py -d stat_files
Finished command 'shell.exec' in function 'upload stat files' (step 6.1 of 9) in block 'post' in 54.647903ms.
Running command 'archive.targz_pack' in function 'upload stat files' (step 6.2 of 9) in block 'post'.
Beginning to build archive.
[message='successfully created archive' num_files='42' target='/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger/cmake_build/stat_files.tar.gz']
Finished command 'archive.targz_pack' in function 'upload stat files' (step 6.2 of 9) in block 'post' in 24.291978ms.
Running command 's3.put' in function 'upload stat files' (step 6.3 of 9) in block 'post'.
Putting local file 'wiredtiger/cmake_build/stat_files.tar.gz' into path 'build_external/wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/stat_files.tar.gz' (https://build_external.s3.amazonaws.com/wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/stat_files.tar.gz).
Performing S3 put to file 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38/stat_files.tar.gz' in bucket 'build_external' (attempt 1 of 5).
Finished command 's3.put' in function 'upload stat files' (step 6.3 of 9) in block 'post' in 340.9655ms.
Running command 'archive.targz_pack' in function 'upload artifact' (step 7.1 of 9) in block 'post'.
Beginning to build archive.
[message='successfully created archive' num_files='8477' target='/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wiredtiger.tgz']
Finished command 'archive.targz_pack' in function 'upload artifact' (step 7.1 of 9) in block 'post' in 1.252943093s.
Running command 's3.put' in function 'upload artifact' (step 7.2 of 9) in block 'post'.
Putting local file 'wiredtiger.tgz' into path 'build_external/wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38-0.tgz' (https://build_external.s3.amazonaws.com/wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38-0.tgz).
Performing S3 put to file 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/artifacts/schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38-0.tgz' in bucket 'build_external' (attempt 1 of 5).
Finished command 's3.put' in function 'upload artifact' (step 7.2 of 9) in block 'post' in 1.201031823s.
Running command 'archive.targz_pack' in function 'save wt hang analyzer core/debugger files' (step 8.1 of 9) in block 'post'.
Beginning to build archive.
No files were archived.
Finished command 'archive.targz_pack' in function 'save wt hang analyzer core/debugger files' (step 8.1 of 9) in block 'post' in 329.765µs.
Running command 's3.put' in function 'save wt hang analyzer core/debugger files' (step 8.2 of 9) in block 'post'.
Putting local file 'wt-hang-analyzer.tgz' into path 'build_external/wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/wt_hang_analyzer/wt-hang-analyzer_schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38.tgz' (https://build_external.s3.amazonaws.com/wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/wt_hang_analyzer/wt-hang-analyzer_schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38.tgz).
Performing S3 put to file 'wiredtiger/amazon2023-disagg-asan-stress/f85479d5032388082a3142d717f631fd7deb39bd/wt_hang_analyzer/wt-hang-analyzer_schema-disagg-abort-smoke-test-disagg-1_wiredtiger_disagg_amazon2023_disagg_asan_stress_f85479d5032388082a3142d717f631fd7deb39bd_26_07_30_10_58_38.tgz' in bucket 'build_external' (attempt 1 of 5).
File '/data/mci/ba300d3d466c2a95109fe0c8f79cd817/wt-hang-analyzer.tgz' not found and skip missing is true, exiting without error.
Finished command 's3.put' in function 'save wt hang analyzer core/debugger files' (step 8.2 of 9) in block 'post' in 10.000733ms.
Running command 'shell.exec' in function 'cleanup' (step 9 of 9) in block 'post'.
Finished command 'shell.exec' in function 'cleanup' (step 9 of 9) in block 'post' in 343.081623ms.
Finished running post-task commands in 5.791614999s.
reading test log spec; falling back to default spec: open /data/mci/ba300d3d466c2a95109fe0c8f79cd817/build/TestLogs/log_spec.yaml: no such file or directory