test/format (disagg.mode=switch) heap-use-after-free when comparing keys in disagg

XMLWordPrintableJSON

      format-stress-test-disagg-switch-1 on amazon2023-disagg-asan-stress

      Host: i-0afd6fb9de4a6f438
      Project: wiredtiger
      Commit: 4d9df4cd
      Please refer to BF(G) Playbook for instructions on handling BF and BFG tickets as well as Auto-Resolution Rules

      Task Logs:

      format-stress-test-disagg-switch-1 task_log

      Logs:

      ==4392==ERROR: AddressSanitizer: heap-use-after-free on address 0x50400129cd50 at pc 0xffffb2983374 bp 0xffff03032250 sp 0xffff03032248
      READ of size 16 at 0x50400129cd50 thread T79
          #0 0xffffb2983370 in __wt_lex_compare /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/btree_cmp_inline.h:81:17
          #1 0xffffb2979b40 in __wt_compare /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/btree_cmp_inline.h:110:17
          #2 0xffffb2978f9c in __check_leaf_key_range /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/row_srch.c:326:9
          #3 0xffffb2975d2c in __wt_row_search /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/row_srch.c:406:13
          #4 0xffffb2700514 in __cursor_row_search /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_cursor.c:573:5
          #5 0xffffb2702d18 in __btcur_search_near_row_pinned_page /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_cursor.c:955:5
          #6 0xffffb2701920 in __wt_btcur_search_near /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_cursor.c:1023:5
          #7 0xffffb2b051c8 in __wti_curfile_search_near /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_file.c:352:5
          #8 0xffffb2bb11a8 in __clayered_position_constituent /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:717:5
          #9 0xffffb2ba8fa8 in __clayered_iterate_constituent /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:769:14
          #10 0xffffb2bb1ca0 in __layered_prev_int /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:866:9
          #11 0xffffb2b863f4 in __layered_prev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:924:5
          #12 0xaaaad426d4c4 in read_op /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/format_inline.h:48:15
          #13 0xaaaad426a800 in nextprev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:1748:16
          #14 0xaaaad425d0d4 in table_op /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:943:24
          #15 0xaaaad42540e8 in ops /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:1346:19
          #16 0xaaaad41ddb80 in asan_thread_start(void*) /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:239:28
          #17 0xffffb22bdb74 in start_thread (/lib64/libc.so.6+0x8bb74) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #18 0xffffb232acd8 in thread_start (/lib64/libc.so.6+0xf8cd8) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
      0x50400129cd50 is located 0 bytes inside of 33-byte region [0x50400129cd50,0x50400129cd71)
      freed by thread T79 here:
          #0 0xaaaad41e00a4 in free /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_malloc_linux.cpp:52:3
          #1 0xffffb2dc58a4 in __wt_free_int /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/os_common/os_alloc.c:274:5
          #2 0xffffb27107c8 in __wt_buf_free /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/buf_inline.h:122:5
          #3 0xffffb2710734 in __wt_btcur_free_cached_memory /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_cursor.c:2281:5
          #4 0xffffb2710828 in __wt_btcur_close /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_cursor.c:2304:5
          #5 0xffffb2b2a89c in __curfile_close /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_file.c:639:5
          #6 0xffffb2bab928 in __clayered_adjust_state /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:488:13
          #7 0xffffb2ba881c in __clayered_enter /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:135:5
          #8 0xffffb2bb1b74 in __layered_prev_int /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:862:5
          #9 0xffffb2b863f4 in __layered_prev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:924:5
          #10 0xaaaad426d4c4 in read_op /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/format_inline.h:48:15
          #11 0xaaaad426a800 in nextprev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:1748:16
          #12 0xaaaad425d0d4 in table_op /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:943:24
          #13 0xaaaad42540e8 in ops /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:1346:19
          #14 0xaaaad41ddb80 in asan_thread_start(void*) /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:239:28
          #15 0x3cffffb232acd8  (<unknown module>)
      previously allocated by thread T79 here:
          #0 0xaaaad41e06c4 in realloc /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_malloc_linux.cpp:82:3
          #1 0xffffb2dc4c64 in __realloc_func /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/os_common/os_alloc.c:160:18
          #2 0xffffb2dc4e80 in __wt_realloc_noclear /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/os_common/os_alloc.c:198:13
          #3 0xffffb30bb5f0 in __wt_buf_grow_worker /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/support/scratch.c:52:9
          #4 0xffffb2946290 in __wt_buf_grow /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/buf_inline.h:24:9
          #5 0xffffb2946028 in __wt_buf_init /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/buf_inline.h:57:13
          #6 0xffffb29439b0 in __wt_row_leaf_key_work /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/row_key.c:255:17
          #7 0xffffb26eb9d8 in __cursor_row_slot_key_return /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/cursor_inline.h:562:9
          #8 0xffffb26d0c2c in __cursor_row_prev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_curprev.c:688:9
          #9 0xffffb26c78c8 in __wt_btcur_prev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_curprev.c:841:23
          #10 0xffffb2b168bc in __curfile_prev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_file.c:245:5
          #11 0xffffb2bb21d0 in __layered_prev_int /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:893:9
          #12 0xffffb2b863f4 in __layered_prev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:924:5
          #13 0xaaaad426d4c4 in read_op /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/format_inline.h:48:15
          #14 0xaaaad426a800 in nextprev /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:1748:16
          #15 0xaaaad425d0d4 in table_op /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:943:24
          #16 0xaaaad42540e8 in ops /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:1346:19
          #17 0xaaaad41ddb80 in asan_thread_start(void*) /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:239:28
          #18 0x3cffffb232acd8  (<unknown module>)
      Thread T79 created by T0 here:
          #0 0xaaaad41c4f90 in pthread_create /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:250:3
          #1 0xffffb2dee994 in __wt_thread_create /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/os_posix/os_thread.c:60:5
          #2 0xaaaad424ebb8 in operations /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:361:9
          #3 0xaaaad427cc60 in main /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/t.c:398:13
          #4 0xffffb2262554 in __libc_start_call_main (/lib64/libc.so.6+0x30554) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #5 0x23ffffb2262638  (<unknown module>)
          #6 0x66aaaad414082c  (<unknown module>)
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      SUMMARY: AddressSanitizer: heap-use-after-free /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/btree_cmp_inline.h:81:17 in __wt_lex_compare
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      Shadow bytes around the buggy address:
        0x50400129ca80: fa fa fd fd fd fd fd fd fa fa fd fd fd fd fd fa
        0x50400129cb00: fa fa fd fd fd fd fd fa fa fa fd fd fd fd fd fa
        0x50400129cb80: fa fa fd fd fd fd fd fa fa fa 00 00 00 00 01 fa
        0x50400129cc00: fa fa fd fd fd fd fd fa fa fa 00 00 00 00 05 fa
        0x50400129cc80: fa fa 00 00 00 00 00 fa fa fa fd fd fd fd fd fa
      =>0x50400129cd00: fa fa fd fd fd fd fd fa fa fa[fd]fd fd fd fd fa
        0x50400129cd80: fa fa fd fd fd fd fd fa fa fa fd fd fd fd fd fa
        0x50400129ce00: fa fa 00 00 00 00 00 05 fa fa 00 00 00 00 00 fa
        0x50400129ce80: fa fa 00 00 00 00 00 fa fa fa 00 00 00 00 00 fa
        0x50400129cf00: fa fa 00 00 00 00 00 fa fa fa fd fd fd fd fd fd
        0x50400129cf80: fa fa 00 00 00 00 00 fa fa fa 00 00 00 00 00 fa
      Shadow byte legend (one shadow byte represents 8 application bytes):
        Addressable:           00
        Partially addressable: 01 02 03 04 05 06 07
        Heap left redzone:       fa
        Freed heap region:       fd
        Stack left redzone:      f1
        Stack mid redzone:       f2
        Stack right redzone:     f3
        Stack after return:      f5
        Stack use after scope:   f8
        Global redzone:          f9
        Global init order:       f6
        Poisoned by user:        f7
        Container overflow:      fc
        Array cookie:            ac
        Intra object redzone:    bb
        ASan internal:           fe
        Left alloca redzone:     ca
        Right alloca redzone:    cb
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      #0  0x0000ffffb22bf7b4 in __pthread_kill_implementation () from /lib64/libc.so.6
      #0  0x0000ffffb22bf7b4 in __pthread_kill_implementation () from /lib64/libc.so.6
      #1  0x0000ffffb22763a0 [PAC] in raise () from /lib64/libc.so.6
      #2  0x0000ffffb2262264 [PAC] in abort () from /lib64/libc.so.6
      #3  0x0000aaaad4205d9c [PAC] in Abort () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp:163
      #4  0x0000aaaad4203c84 in __sanitizer::Die() () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/sanitizer_common/sanitizer_termination.cpp:58
      #5  0x0000aaaad41e51c8 in ~ScopedInErrorReport () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_report.cpp:193
      #6  0x0000aaaad41e8508 in ReportGenericError () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_report.cpp:498
      #7  0x0000aaaad41e9a00 in __asan_report_load_n () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_rtl.cpp:156
      #8  0x0000ffffb2983374 in __wt_lex_compare (user_item=0x51a0018b29a0, tree_item=0x51a0018b2b98) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/btree_cmp_inline.h:81
      #9  0x0000ffffb2979b44 in __wt_compare (session=0xffffb16b9558, collator=0x0, user_item=0x51a0018b29a0, tree_item=0x51a0018b2b98, cmpp=0xffff030326e0) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/include/btree_cmp_inline.h:110
      #10 0x0000ffffb2978fa0 in __check_leaf_key_range (session=0xffffb16b9558, srch_key=0x51a0018b29a0, leaf=0x50e001d4fcc0, cbt=0x51a0018b2880) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/row_srch.c:326
      #11 0x0000ffffb2975d30 in __wt_row_search (cbt=0x51a0018b2880, srch_key=0x51a0018b29a0, insert=true, leaf=0x50e001d4fcc0, leaf_safe=false, leaf_foundp=0xfffeffce9ba0) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/row_srch.c:406
      #12 0x0000ffffb2700518 in __cursor_row_search (cbt=0x51a0018b2880, insert=true, leaf=0x50e001d4fcc0, leaf_foundp=0xfffeffce9ba0) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_cursor.c:573
      #13 0x0000ffffb2702d1c in __btcur_search_near_row_pinned_page (cbt=0x51a0018b2880, validp=0xfffeffeeb1c0) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_cursor.c:955
      #14 0x0000ffffb2701924 in __wt_btcur_search_near (cbt=0x51a0018b2880, exactp=0xfffeffce9b20) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/btree/bt_cursor.c:1023
      #15 0x0000ffffb2b051cc in __wti_curfile_search_near (cursor=0x51a0018b2880, exact=0xfffeffce9b20) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_file.c:352
      #16 0x0000ffffb2bb11ac in __clayered_position_constituent (clayered=0x516001536f80, c=0x51a0018b2880, forward=true, cmpp=0xfffeffce9b20) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:717
      #17 0x0000ffffb2ba8fac in __clayered_iterate_constituent (clayered=0x516001536f80, constituent=0x51a0018b2880, forward=false) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:769
      #18 0x0000ffffb2bb1ca4 in __layered_prev_int (session=0xffffb16b9558, cursor=0x516001536f80) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:866
      #19 0x0000ffffb2b863f8 in __layered_prev (cursor=0x516001536f80) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/src/cursor/cur_layered.c:924
      #20 0x0000aaaad426d4c8 in read_op (cursor=0x516001536f80, op=PREV, exactp=0x0) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/format_inline.h:48
      #21 0x0000aaaad426a804 in nextprev (tinfo=0x519000142380, next=false) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:1748
      #22 0x0000aaaad425d0d8 in table_op (tinfo=0x519000142380, intxn=true, iso_level=ISOLATION_SNAPSHOT, op=UPDATE) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:943
      #23 0x0000aaaad42540ec in ops (arg=0x519000142380) at /data/mci/a03f9f84d4be21aa51ec5bc7dc667a4f/wiredtiger/test/format/ops.c:1346
      #24 0x0000aaaad41ddb84 in asan_thread_start () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:239
      #25 0x0000ffffb22bdb78 in start_thread () from /lib64/libc.so.6
      #26 0x0000ffffb232acdc [PAC] in thread_start () from /lib64/libc.so.6
      

      logs

      Repro Artifacts:

            Assignee:
            [DO NOT USE] Backlog - Storage Engines Team
            Reporter:
            xgen-buildbaron-user
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated: