Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-3281

Fix a bug where we attempt to copy a key out of a page after it's evicted

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • WT2.9.3, 3.5.9
    • Affects Version/s: None
    • Component/s: None
    • Labels:
      None
    • Storage 2017-04-17, Storage 2017-05-08

      Another stress test sanitizer failure.

      ==18563==ERROR: AddressSanitizer: heap-use-after-free on address 0x6040006e06e4 at pc 0x9e1e96 bp 0x7f86babc4b10 sp 0x7f86babc4b08
      READ of size 15 at 0x6040006e06e4 thread T15
          #0 0x9e1e95 in __wt_buf_grow_worker /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/support/scratch.c:57
          #1 0x97cf1e in __wt_buf_grow /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/buf.i:18
          #2 0x91fad5 in __wt_buf_set /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/buf.i:85:10
          #3 0x91f3e8 in __wt_session_copy_values /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/session/session_api.c:77:34
          #4 0xbda4c9 in __wt_txn_begin /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/txn.i:268
          #5 0xbd9a50 in __wt_txn_autocommit_check /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/txn.i:297:11
          #6 0xbd3864 in __wt_page_in_func /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_read.c:647:8
          #7 0xd4b6d3 in __wt_page_swap_func /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/btree.i:1427
          #8 0xd45fc3 in __wt_col_search /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/col_srch.c:195:14
          #9 0x10eb089 in __cursor_col_search /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:343
          #10 0x10f95a2 in __wt_btcur_remove /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:931:20
          #11 0xe48cd9 in __curfile_remove /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/cursor/cur_file.c:319
          #12 0x4aba1b in col_remove /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1535
          #13 0x49fdfd in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:724
          #14 0x7f86cc248dc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)
          #15 0x7f86cb42c73c in __clone (/lib64/libc.so.6+0xf773c)
      
      0x6040006e06e4 is located 20 bytes inside of 35-byte region [0x6040006e06d0,0x6040006e06f3)
      freed by thread T15 here:
          #0 0x467f49 in __interceptor_free (/mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/t+0x467f49)
          #1 0x753420 in __wt_free_int /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_common/os_alloc.c:309
          #2 0xb41527 in __wt_free_update_list /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_discard.c:463
          #3 0xb4a1ef in __free_skip_list /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_discard.c:422
          #4 0xb46da6 in __free_page_modify /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_discard.c:220
          #5 0xb3e1f6 in __page_out_int /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_discard.c:137
          #6 0xb3ee46 in __wt_page_out /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_discard.c:168
          #7 0xc79690 in __split_multi /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_split.c:2036
          #8 0xc77568 in __wt_split_multi /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_split.c:2061
          #9 0x64893b in __evict_page_dirty_update /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_page.c:330
          #10 0x642508 in __wt_evict /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_page.c:170
          #11 0x63f868 in __wt_page_release_evict /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/evict/evict_page.c:78
          #12 0x1112b7a in __wt_page_release /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/btree.i:1393
          #13 0x10e686d in __cursor_reset /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/cursor.i:194:8
          #14 0x10eb636 in __cursor_func_init /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/include/cursor.i:308:33
          #15 0x10f8fbf in __wt_btcur_remove /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:918:25
          #16 0xe48cd9 in __curfile_remove /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/cursor/cur_file.c:319
          #17 0x4aba1b in col_remove /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1535
          #18 0x49fdfd in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:724
          #19 0x7f86cc248dc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)
      
      previously allocated by thread T17 here:
          #0 0x468199 in calloc (/mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/t+0x468199)
          #1 0x74eaa1 in __wt_calloc /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/os_common/os_alloc.c:52
          #2 0xd716d7 in __wt_update_alloc /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/row_modify.c:281
          #3 0xd2b132 in __wt_col_modify /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/col_modify.c:152
          #4 0x10f5653 in __cursor_col_modify /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:371
          #5 0x10f244f in __wt_btcur_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/btree/bt_cursor.c:738:20
          #6 0xe4428c in __curfile_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/cursor/cur_file.c:240
          #7 0x4a9b94 in col_insert /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:1430
          #8 0x49ef47 in ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:683
          #9 0x7f86cc248dc4 in start_thread (/lib64/libpthread.so.0+0x7dc4)
      
      Thread T15 created by T0 here:
          #0 0x457582 in __interceptor_pthread_create (/mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/t+0x457582)
          #1 0x497607 in wts_ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:120
          #2 0x4b973d in main /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/t.c:209
          #3 0x7f86cb356b34 in __libc_start_main (/lib64/libc.so.6+0x21b34)
      
      Thread T17 created by T0 here:
          #0 0x457582 in __interceptor_pthread_create (/mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/t+0x457582)
          #1 0x497607 in wts_ops /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/ops.c:120
          #2 0x4b973d in main /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/test/format/../../../test/format/t.c:209
          #3 0x7f86cb356b34 in __libc_start_main (/lib64/libc.so.6+0x21b34)
      
      SUMMARY: AddressSanitizer: heap-use-after-free /mnt/data0/jenkins/workspace/wiredtiger-test-format-stress-sanitizer/build_posix/../src/support/scratch.c:57 __wt_buf_grow_worker
      Shadow bytes around the buggy address:
        0x0c08800d4080: fa fa fd fd fd fd fd fd fa fa fd fd fd fd fd fa
        0x0c08800d4090: fa fa fd fd fd fd fd fd fa fa fd fd fd fd fd fd
        0x0c08800d40a0: fa fa fd fd fd fd fd fa fa fa fd fd fd fd fd fa
        0x0c08800d40b0: fa fa fd fd fd fd fd fd fa fa fd fd fd fd fd fd
        0x0c08800d40c0: fa fa fd fd fd fd fd fd fa fa fd fd fd fd fd fd
      =>0x0c08800d40d0: fa fa 00 00 00 00 00 03 fa fa fd fd[fd]fd fd fa
        0x0c08800d40e0: fa fa fd fd fd fd fd fd fa fa fd fd fd fd fd fd
        0x0c08800d40f0: fa fa fd fd fd fd fd fa fa fa 00 00 00 00 00 03
        0x0c08800d4100: fa fa 00 00 00 00 00 03 fa fa fd fd fd fd fd fd
        0x0c08800d4110: fa fa fd fd fd fd fd fd fa fa fd fd fd fd fd fd
        0x0c08800d4120: fa fa fd fd fd fd fd fd fa fa fd fd fd fd fd fd
      Shadow byte legend (one shadow byte represents 8 application bytes):
        Addressable:           00
        Partially addressable: 01 02 03 04 05 06 07
        Heap left redzone:     fa
        Heap right redzone:    fb
        Freed heap region:     fd
        Stack left redzone:    f1
        Stack mid redzone:     f2
        Stack right redzone:   f3
        Stack partial redzone: f4
        Stack after return:    f5
        Stack use after scope: f8
        Global redzone:        f9
        Global init order:     f6
        Poisoned by user:      f7
        ASan internal:         fe
      ==18563==ABORTING
      + cleanup
      + status=1
      + '[' -f RUNDIR/CONFIG ']'
      + cat RUNDIR/CONFIG
      ############################################
      #  RUN PARAMETERS
      ############################################
      abort=0
      alter=0
      auto_throttle=0
      backups=0
      bitcnt=7
      bloom=1
      bloom_bit_count=26
      bloom_hash_count=6
      bloom_oldest=0
      cache=94
      checkpoints=1
      checksum=uncompressed
      chunk_size=4
      compaction=0
      compression=zlib
      data_extend=0
      data_source=table
      delete_pct=22
      dictionary=0
      direct_io=0
      encryption=none
      evict_max=3
      file_type=variable-length column-store
      firstfit=0
      huffman_key=0
      huffman_value=0
      in_memory=0
      insert_pct=38
      internal_key_truncation=1
      internal_page_max=11
      isolation=snapshot
      key_gap=11
      key_max=61
      key_min=15
      leaf_page_max=9
      leak_memory=0
      logging=1
      logging_archive=0
      logging_compression=none
      logging_prealloc=0
      long_running_txn=0
      lsm_worker_threads=3
      merge_max=8
      mmap=1
      ops=100000
      prefix_compression=0
      prefix_compression_min=2
      quiet=1
      read_pct=29
      rebalance=1
      repeat_data_pct=58
      reverse=1
      rows=100000
      runs=1
      salvage=1
      split_pct=88
      statistics=0
      statistics_server=0
      threads=18
      timer=20
      transaction-frequency=61
      value_max=3499
      value_min=10
      verify=1
      wiredtiger_config=
      write_pct=11
      ############################################
      

            Assignee:
            keith.bostic@mongodb.com Keith Bostic (Inactive)
            Reporter:
            keith.bostic@mongodb.com Keith Bostic (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

              Created:
              Updated:
              Resolved: