test/format (disagg.mode=switch) heap-use-after-free ASAN bug

XMLWordPrintableJSON

      format-stress-test-disagg-switch-1 on amazon2023-disagg-asan-stress

      Host: i-0feaa39e67b7dee61
      Project: wiredtiger
      Commit: bed4db7b
      Please refer to BF(G) Playbook for instructions on handling BF and BFG tickets as well as Auto-Resolution Rules

      Task Logs:

      format-stress-test-disagg-switch-1 task_log

      Logs:

      ==28700==ERROR: AddressSanitizer: heap-use-after-free on address 0x50300067e770 at pc 0xffff9ba965c4 bp 0xffffeb213fd0 sp 0xffffeb213fc8
      READ of size 8 at 0x50300067e770 thread T0
          #0 0xffff9ba965c0 in __layered_drain_ingest_tables /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1964:13
          #1 0xffff9ba87514 in __disagg_step_up /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1206:5
          #2 0xffff9ba83bfc in __wti_disagg_conn_config /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1306:9
          #3 0xffff9bab6544 in __wti_conn_reconfig /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_reconfig.c:450:13
          #4 0xffff9ba228b4 in __conn_reconfigure /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_api.c:1354:11
          #5 0xaaaaabf42b9c in disagg_switch_roles /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/test/format/format_disagg.c:163:5
          #6 0xaaaaabf7be88 in main /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/test/format/t.c:408:13
          #7 0xffff9b262554 in __libc_start_call_main (/lib64/libc.so.6+0x30554) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #8 0xffff9b262638 in __libc_start_main@GLIBC_2.17 (/lib64/libc.so.6+0x30638) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #9 0xaaaaabf1f2ec in _start (/data/mci/c901f19fa0522159f08ff3a30382b7ed/wiredtiger/cmake_build/test/format/t+0x2f2ec)
      0x50300067e770 is located 16 bytes inside of 26-byte region [0x50300067e760,0x50300067e77a)
      freed by thread T0 here:
          #0 0xffff9c907d00 in free /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_malloc_linux.cpp:52:3
          #1 0xffff9bdf270c in __wt_free_int /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/os_common/os_alloc.c:274:5
          #2 0xffff9b7d9c10 in __wt_buf_free /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/include/buf_inline.h:122:5
          #3 0xffff9b7c2354 in __wt_cell_pack_leaf_kv /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/include/cell_inline.h:412:5
          #4 0xffff9b7bd524 in __wti_page_merge_deltas_with_base_image_leaf /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/bt_page.c:852:13
          #5 0xffff9b828dc4 in __page_read_build_full_disk_image /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/bt_read.c:166:9
          #6 0xffff9b8218c4 in __page_read /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/bt_read.c:326:9
          #7 0xffff9b81dee8 in __wt_page_in_func /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/bt_read.c:538:13
          #8 0xffff9b99e4a8 in __wt_page_swap_func /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/include/btree_inline.h:2580:11
          #9 0xffff9b999aac in __wt_row_search /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/row_srch.c:576:20
          #10 0xffff9ba9a124 in __layered_move_updates /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1713:5
          #11 0xffff9ba98098 in __layered_copy_ingest_table /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1822:17
          #12 0xffff9ba963f8 in __layered_drain_ingest_tables /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1961:13
          #13 0xffff9ba87514 in __disagg_step_up /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1206:5
          #14 0xffff9ba83bfc in __wti_disagg_conn_config /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1306:9
          #15 0xffff9bab6544 in __wti_conn_reconfig /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_reconfig.c:450:13
          #16 0xffff9ba228b4 in __conn_reconfigure /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_api.c:1354:11
          #17 0xaaaaabf42b9c in disagg_switch_roles /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/test/format/format_disagg.c:163:5
          #18 0xaaaaabf7be88 in main /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/test/format/t.c:408:13
          #19 0xffff9b262554 in __libc_start_call_main (/lib64/libc.so.6+0x30554) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #20 0x53ffff9b262638  (<unknown module>)
          #21 0x66aaaaabf1f2ec  (<unknown module>)
      previously allocated by thread T0 here:
          #0 0xffff9c908320 in realloc /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_malloc_linux.cpp:82:3
          #1 0xffff9bdf1af8 in __realloc_func /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/os_common/os_alloc.c:160:18
          #2 0xffff9bdf1d14 in __wt_realloc_noclear /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/os_common/os_alloc.c:198:13
          #3 0xffff9c0da154 in __wt_buf_grow_worker /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/support/scratch.c:52:9
          #4 0xffff9b7d8378 in __wt_buf_grow /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/include/buf_inline.h:24:9
          #5 0xffff9b7d8cf0 in __wt_buf_set /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/include/buf_inline.h:88:13
          #6 0xffff9b7c1c3c in __wt_cell_pack_leaf_kv /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/include/cell_inline.h:370:5
          #7 0xffff9b7bd524 in __wti_page_merge_deltas_with_base_image_leaf /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/bt_page.c:852:13
          #8 0xffff9b828dc4 in __page_read_build_full_disk_image /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/bt_read.c:166:9
          #9 0xffff9b8218c4 in __page_read /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/bt_read.c:326:9
          #10 0xffff9b81dee8 in __wt_page_in_func /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/bt_read.c:538:13
          #11 0xffff9b99e4a8 in __wt_page_swap_func /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/include/btree_inline.h:2580:11
          #12 0xffff9b999aac in __wt_row_search /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/btree/row_srch.c:576:20
          #13 0xffff9ba9a124 in __layered_move_updates /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1713:5
          #14 0xffff9ba98098 in __layered_copy_ingest_table /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1822:17
          #15 0xffff9ba963f8 in __layered_drain_ingest_tables /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1961:13
          #16 0xffff9ba87514 in __disagg_step_up /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1206:5
          #17 0xffff9ba83bfc in __wti_disagg_conn_config /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1306:9
          #18 0xffff9bab6544 in __wti_conn_reconfig /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_reconfig.c:450:13
          #19 0xffff9ba228b4 in __conn_reconfigure /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_api.c:1354:11
          #20 0xaaaaabf42b9c in disagg_switch_roles /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/test/format/format_disagg.c:163:5
          #21 0xaaaaabf7be88 in main /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/test/format/t.c:408:13
          #22 0xffff9b262554 in __libc_start_call_main (/lib64/libc.so.6+0x30554) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #23 0x53ffff9b262638  (<unknown module>)
          #24 0x66aaaaabf1f2ec  (<unknown module>)
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      SUMMARY: AddressSanitizer: heap-use-after-free /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1964:13 in __layered_drain_ingest_tables
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      Shadow bytes around the buggy address:
        0x50300067e480: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x50300067e500: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x50300067e580: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x50300067e600: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x50300067e680: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
      =>0x50300067e700: fa fa fa fa fa fa fa fa fa fa fa fa fd fd[fd]fd
        0x50300067e780: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x50300067e800: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x50300067e880: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x50300067e900: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x50300067e980: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
      Shadow byte legend (one shadow byte represents 8 application bytes):
        Addressable:           00
        Partially addressable: 01 02 03 04 05 06 07
        Heap left redzone:       fa
        Freed heap region:       fd
        Stack left redzone:      f1
        Stack mid redzone:       f2
        Stack right redzone:     f3
        Stack after return:      f5
        Stack use after scope:   f8
        Global redzone:          f9
        Global init order:       f6
        Poisoned by user:        f7
        Container overflow:      fc
        Array cookie:            ac
        Intra object redzone:    bb
        ASan internal:           fe
        Left alloca redzone:     ca
        Right alloca redzone:    cb
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      #0  0x0000ffff9b2bf7b4 in __pthread_kill_implementation () from /lib64/libc.so.6
      #0  0x0000ffff9b2bf7b4 in __pthread_kill_implementation () from /lib64/libc.so.6
      #1  0x0000ffff9b2763a0 [PAC] in raise () from /lib64/libc.so.6
      #2  0x0000ffff9b262264 [PAC] in abort () from /lib64/libc.so.6
      #3  0x0000ffff9c85133c [PAC] in Abort () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp:163
      #4  0x0000ffff9c84f224 in __sanitizer::Die() () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/sanitizer_common/sanitizer_termination.cpp:58
      #5  0x0000ffff9c90cb80 in ~ScopedInErrorReport () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_report.cpp:193
      #6  0x0000ffff9c90fec0 in ReportGenericError () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_report.cpp:498
      #7  0x0000ffff9c910dd0 in __asan_report_load8 () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_rtl.cpp:131
      #8  0x0000ffff9ba965c4 in __layered_drain_ingest_tables (session=0xffff9a69e800) at /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1964
      #9  0x0000ffff9ba87518 in __disagg_step_up (session=0xffff9a69e800) at /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1206
      #10 0x0000ffff9ba83c00 in __wti_disagg_conn_config (session=0xffff9a69e800, cfg=0xffff99606020, reconfig=true) at /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_layered.c:1306
      #11 0x0000ffff9bab6548 in __wti_conn_reconfig (session=0xffff9a69e800, cfg=0xffff99606020) at /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_reconfig.c:450
      #12 0x0000ffff9ba228b8 in __conn_reconfigure (wt_conn=0x525000ee7100, config=0xffff991dbc20 "disaggregated=(role=\"leader\")") at /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/src/conn/conn_api.c:1354
      #13 0x0000aaaaabf42ba0 in disagg_switch_roles () at /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/test/format/format_disagg.c:163
      #14 0x0000aaaaabf7be8c in main (argc=5, argv=0xffffeb215bb0) at /data/mci/9198eb838774883b734f2f90b340d20b/wiredtiger/test/format/t.c:408
      

      logs

      Repro Artifacts:

            Assignee:
            Chenhao Qu
            Reporter:
            xgen-buildbaron-user
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: