Loading...

Type: Bug
Resolution: Fixed
Priority: Major - P3
Fix Version/s: WT3.2.1, 4.3.1, 4.2.0-rc1
Affects Version/s: None
Component/s: None
Labels:
- format-cache-stuck

Sprint:
Storage Engines 2019-06-03
Story Points:
8

Backport Requested:

v4.2

This LSM cache stuck clean case does not look the same as ~~WT-3694~~. The stress sanitizer job has 2 round of tests, one with the other without -enable-page-version-ts configured. The test failed at the round -enable-page-version-ts being configured. Also there's no page was dumped out in this case.

http://build.wiredtiger.com:8080/job/wiredtiger-test-format-stress-sanitizer/23730/

+ ../configure 'CC=clang -fsanitize=address' 'CFLAGS=-g -fno-omit-frame-pointer' --enable-diagnostic --enable-page-version-ts --with-builtins=lz4,snappy,zlib --with-berkeleydb=/usr/local/BerkeleyDB.6.0

(...)

++ nice ./t -1 -c ../../../test/format/CONFIG.stress file_type=row leaf_page_max=9 internal_page_max=9 key_min=256 value_min=256
[1551255293:404305][23905:0x7fd2c6c16700], t, eviction-server: __evict_server, 451: Cache stuck for too long, giving up: Connection timed out
t: process 23905
0x7fd2cecb9690:=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
0x7fd2cecb9690:transaction state dump
0x7fd2cecb9690:current ID: 9
0x7fd2cecb9690:last running ID: 9
0x7fd2cecb9690:metadata_pinned ID: 9
0x7fd2cecb9690:oldest ID: 9
0x7fd2cecb9690:commit timestamp: (0,0)
0x7fd2cecb9690:oldest timestamp: (0,0)
0x7fd2cecb9690:pinned timestamp: (0,0)
0x7fd2cecb9690:stable timestamp: (0,0)
0x7fd2cecb9690:has_commit_timestamp: no
0x7fd2cecb9690:has_oldest_timestamp: no
0x7fd2cecb9690:has_pinned_timestamp: no
0x7fd2cecb9690:has_stable_timestamp: no
0x7fd2cecb9690:oldest_is_pinned: no
0x7fd2cecb9690:stable_is_pinned: no
0x7fd2cecb9690:checkpoint running: no
0x7fd2cecb9690:checkpoint generation: 1
0x7fd2cecb9690:checkpoint pinned ID: 0
0x7fd2cecb9690:checkpoint txn ID: 0
0x7fd2cecb9690:oldest named snapshot ID: 0
0x7fd2cecb9690:session count: 33
0x7fd2cecb9690:Transaction state of active sessions:
0x7fd2cecb9690:=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
0x7fd2cecb9690:cache dump
0x7fd2cecb9690:cache full: yes
0x7fd2cecb9690:cache clean check: yes (298.904%)
0x7fd2cecb9690:cache dirty check: no (0.002%)
ASAN:DEADLYSIGNAL
=================================================================
==23905==ERROR: AddressSanitizer: SEGV on unknown address 0x000000000020 (pc 0x0000008bf660 bp 0x7fd2c6c14ef0 sp 0x7fd2c6c149e0 T4)
==23905==The signal is caused by a READ memory access.
==23905==Hint: address points to the zero page.
    #0 0x8bf65f in __wt_page_in_func /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_read.c:809:23
    #1 0x95fa97 in __wt_page_swap_func /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/btree.i:1702:8
    #2 0x95d984 in __tree_walk_internal /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_walk.c:526:10
    #3 0x95c272 in __wt_tree_walk /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_walk.c:597:10
    #4 0x60c9eb in __verbose_dump_cache_single /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:2561:9
    #5 0x5f5e70 in __verbose_dump_cache_apply /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:2666:3
    #6 0x5f5230 in __wt_verbose_dump_cache /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:2704:2
    #7 0x5eba01 in __evict_server /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:453:3
    #8 0x5eabbe in __wt_evict_thread_run /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:290:9
    #9 0x7bfee3 in __thread_run /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/support/thread_group.c:31:3
    #10 0x7fd2ce252dc4 in start_thread /usr/src/debug/glibc-2.17-c758a686/nptl/pthread_create.c:308
    #11 0x7fd2cd43976c in __clone (/lib64/libc.so.6+0xf776c)

AddressSanitizer can not provide additional info.
SUMMARY: AddressSanitizer: SEGV /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_read.c:809:23 in __wt_page_in_func
Thread T4 created by T0 here:
    #0 0x437511 in pthread_create /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_interceptors.cc:305
    #1 0x672a80 in __wt_thread_create /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_posix/os_thread.c:30:2
    #2 0x7bd5f9 in __thread_group_resize /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/support/thread_group.c:214:3
    #3 0x7be0b0 in __wt_thread_group_create /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/support/thread_group.c:305:2
    #4 0x5ed58f in __wt_evict_create /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:490:2
    #5 0x598f5c in __wt_connection_workers /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/conn/conn_open.c:248:2
    #6 0x5616b2 in wiredtiger_open /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/conn/conn_api.c:2773:2
    #7 0x548ba6 in wts_open /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/wts.c:297:2
    #8 0x53ce9a in main /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/t.c:217:3
    #9 0x7fd2cd363b34 in __libc_start_main /usr/src/debug/glibc-2.17-c758a686/csu/../csu/libc-start.c:274

==23905==ABORTING

The configuration:

############################################
#  RUN PARAMETERS
############################################
abort=0
alter=0
auto_throttle=1
backups=0
bitcnt=1
bloom=1
bloom_bit_count=47
bloom_hash_count=9
bloom_oldest=0
cache=320
cache_minimum=20
checkpoints=on
checkpoint_log_size=163
checkpoint_wait=27
checksum=uncompressed
chunk_size=4
compaction=0
compression=snappy
data_extend=0
data_source=lsm
delete_pct=1
dictionary=0
direct_io=0
encryption=none
evict_max=1
file_type=row-store
firstfit=0
huffman_key=0
huffman_value=0
independent_thread_rng=1
in_memory=0
insert_pct=10
internal_key_truncation=1
internal_page_max=9
isolation=random
key_gap=17
key_max=256
key_min=256
leaf_page_max=9
leak_memory=0
logging=1
logging_archive=1
logging_compression=lz4
logging_file_max=206034
logging_prealloc=0
long_running_txn=0
lsm_worker_threads=4
memory_page_max=5
merge_max=19
mmap=1
modify_pct=30
ops=0
prefix_compression=1
prefix_compression_min=0
prepare=0
quiet=1
read_pct=0
rebalance=1
repeat_data_pct=79
reverse=0
rows=1000000
runs=1
salvage=1
split_pct=81
statistics=0
statistics_server=0
threads=32
timer=4
timing_stress_aggressive_sweep=0
timing_stress_checkpoint=0
timing_stress_lookaside_sweep=0
timing_stress_split_1=0
timing_stress_split_2=0
timing_stress_split_3=0
timing_stress_split_4=0
timing_stress_split_5=0
timing_stress_split_6=0
timing_stress_split_7=0
timing_stress_split_8=1
transaction_timestamps=0
transaction-frequency=84
truncate=0
value_max=412
value_min=256
verify=1
wiredtiger_config=
write_pct=59
############################################

The build_posix directory was archived here.

depends on

WT-4816 Don't disallow LSM configurations in test/format

Closed

is related to

WT-4834 test/format timing stress should be less stressful

Closed

WT-4837 Disabled cache during verify

Closed

related to

WT-4634 segfault in test/format on PPC

Closed

Details

Description

Attachments

Issue Links

Activity

People

Dates