Loading...

Type: Bug
Resolution: Fixed
Priority: Major - P3
Fix Version/s: WT3.0.0
Affects Version/s: None
Component/s: None
Labels:
None

Sprint:
Storage 2017-10-23

lookaside with timestamps: heap-use-after-free

It looks like the truncation of the update list can race with transaction commit when re-building a page in-memory.

Build #17238 origin/wt-3435-lookaside-with-timestamps (Oct 2, 2017 6:51:42 AM)
http://build.wiredtiger.com:8080/job/wiredtiger-test-format-stress-sanitizer/17238/

==17403==ERROR: AddressSanitizer: heap-use-after-free on address 0x60600004413c at pc 0x00000072e149 bp 0x7f29a4082e90 sp 0x7f29a4082e88
READ of size 1 at 0x60600004413c thread T15
    #0 0x72e148 in __wt_txn_commit /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/txn/txn.c:684:19
    #1 0x6ce0a6 in __session_commit_transaction /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/session/session_api.c:1451:9
    #2 0x529381 in commit_transaction /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:498:3
    #3 0x527a2f in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:949:4
    #4 0x7f29b418adc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)
    #5 0x7f29b337176c in __clone (/lib64/libc.so.6+0xf776c)

0x60600004413c is located 28 bytes inside of 53-byte region [0x606000044120,0x606000044155)
freed by thread T15 here:
    #0 0x4df510 in __interceptor_free /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:47
    #1 0x607d19 in __wt_free_int /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_common/os_alloc.c:327:2
    #2 0x87827d in __wt_update_obsolete_free /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/row_modify.c:363:3
    #3 0x8118e5 in __split_multi_inmem /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_split.c:1496:5
    #4 0x81266b in __wt_split_rewrite /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_split.c:2242:2
    #5 0x5cedf2 in __evict_page_dirty_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_page.c:340:4
    #6 0x5cc5a9 in __wt_evict /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_page.c:186:3
    #7 0x5b20e3 in __evict_page /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:2235:2
    #8 0x5b007c in __wt_cache_eviction_worker /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_lru.c:2334:17
    #9 0xa111d8 in __wt_cache_eviction_check /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/cache.i:398:10
    #10 0xa108c6 in __cursor_enter /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/cursor.i:152:3
    #11 0xa00a8c in __cursor_func_init /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/cursor.i:343:4
    #12 0xa032f6 in __wt_btcur_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:731:8
    #13 0x8d574a in __curfile_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/cursor/cur_file.c:240:2
    #14 0x52aa50 in row_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1653:16
    #15 0x5266da in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:778:11
    #16 0x7f29b418adc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)

previously allocated by thread T15 here:
    #0 0x4dfa0d in calloc /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:74
    #1 0x606572 in __wt_calloc /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_common/os_alloc.c:52:11
    #2 0x875af6 in __wt_update_alloc /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/row_modify.c:280:3
    #3 0x873cc9 in __wt_row_modify /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/row_modify.c:94:4
    #4 0xa12dc7 in __cursor_row_modify_v /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:376:10
    #5 0xa08d0c in __btcur_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:1106:9
    #6 0xa0a8c4 in __wt_btcur_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:1342:10
    #7 0x8d63a8 in __curfile_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/cursor/cur_file.c:327:2
    #8 0x52ccc7 in row_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1453:16
    #9 0x5274f4 in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:886:11
    #10 0x7f29b418adc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)

Thread T15 created by T0 here:
    #0 0x437361 in __interceptor_pthread_create /home/bostic/src/llvm40/projects/compiler-rt/lib/asan/asan_interceptors.cc:305
    #1 0x61cc30 in __wt_thread_create /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_posix/os_thread.c:30:2
    #2 0x522466 in wts_ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:162:3
    #3 0x533556 in main /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/t.c:210:5
    #4 0x7f29b329bb34 in __libc_start_main (/lib64/libc.so.6+0x21b34)

SUMMARY: AddressSanitizer: heap-use-after-free /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/txn/txn.c:684:19 in __wt_txn_commit
Shadow bytes around the buggy address:
  0x0c0c800007d0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c0c800007e0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c0c800007f0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c0c80000800: 00 00 00 00 00 00 00 00 fa fa fa fa fa fa fa fa
  0x0c0c80000810: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
=>0x0c0c80000820: fa fa fa fa fd fd fd[fd]fd fd fd fa fa fa fa fa
  0x0c0c80000830: fa fa fa fa fa fa fa fa fa fa fa fa fd fd fd fd
  0x0c0c80000840: fd fd fd fd fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c0c80000850: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c0c80000860: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c0c80000870: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07
  Heap left redzone:       fa
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
==17403==ABORTING

Here's the CONFIG:

############################################
#  RUN PARAMETERS
############################################
abort=0
alter=0
auto_throttle=1
backups=1
bitcnt=3
bloom=1
bloom_bit_count=63
bloom_hash_count=13
bloom_oldest=0
cache=18
checkpoints=1
checksum=uncompressed
chunk_size=9
compaction=0
compression=zlib
data_extend=0
data_source=file
delete_pct=12
dictionary=0
direct_io=0
encryption=none
evict_max=5
file_type=row-store
firstfit=0
huffman_key=0
huffman_value=0
independent_thread_rng=0
in_memory=0
insert_pct=13
internal_key_truncation=1
internal_page_max=12
isolation=random
key_gap=2
key_max=47
key_min=25
leaf_page_max=17
leak_memory=0
logging=1
logging_archive=1
logging_compression=zlib
logging_prealloc=1
long_running_txn=0
lsm_worker_threads=4
merge_max=17
mmap=1
modify_pct=0
ops=100000
prefix_compression=1
prefix_compression_min=2
quiet=1
read_pct=2
rebalance=1
repeat_data_pct=64
reverse=0
rows=100000
runs=1
salvage=1
split_pct=84
statistics=0
statistics_server=0
threads=11
timer=360
transaction_timestamps=0
transaction-frequency=40
value_max=3570
value_min=17
verify=1
wiredtiger_config=
write_pct=73
############################################

is depended on by

WT-3435 Lookaside eviction should be able to save unstable updates

Closed

Details

Description

Attachments

Issue Links

Activity

People

Dates