test/format (disagg.mode=switch) ASAN Heap-use-after-free in __layered_drain_ingest_tables

XMLWordPrintableJSON

      format-stress-test-disagg-switch-1 on amazon2023-disagg-asan-stress

      Host: i-0ad489f0bfa39283c
      Project: wiredtiger
      Commit: 3b91a761
      Please refer to BF(G) Playbook for instructions on handling BF and BFG tickets as well as Auto-Resolution Rules

      Task Logs:

      format-stress-test-disagg-switch-1 task_log

      Logs:

      ==333636==ERROR: AddressSanitizer: heap-use-after-free on address 0x503001a60c90 at pc 0xffff86c965c4 bp 0xffffdd193c90 sp 0xffffdd193c88
      READ of size 8 at 0x503001a60c90 thread T0
          #0 0xffff86c965c0 in __layered_drain_ingest_tables /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_layered.c:1964:13
          #1 0xffff86c87514 in __disagg_step_up /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_layered.c:1206:5
          #2 0xffff86c83bfc in __wti_disagg_conn_config /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_layered.c:1306:9
          #3 0xffff86cb6544 in __wti_conn_reconfig /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_reconfig.c:450:13
          #4 0xffff86c228b4 in __conn_reconfigure /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_api.c:1354:11
          #5 0xaaaae7a02b9c in disagg_switch_roles /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/test/format/format_disagg.c:163:5
          #6 0xaaaae7a3be88 in main /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/test/format/t.c:408:13
          #7 0xffff86462554 in __libc_start_call_main (/lib64/libc.so.6+0x30554) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #8 0xffff86462638 in __libc_start_main@GLIBC_2.17 (/lib64/libc.so.6+0x30638) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #9 0xaaaae79df2ec in _start (/data/mci/a6b90e88cbf1c064c0f27eebcffbc8ba/wiredtiger/cmake_build/test/format/t+0x2f2ec)
      0x503001a60c91 is located 0 bytes after 17-byte region [0x503001a60c80,0x503001a60c91)
      freed by thread T119 here:
          #0 0xffff87b07d00 in free /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_malloc_linux.cpp:52:3
          #1 0xffff86ff270c in __wt_free_int /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/os_common/os_alloc.c:274:5
          #2 0xffff8697fb24 in __free_page_modify /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/btree/bt_discard.c:184:17
          #3 0xffff8697d670 in __wt_page_out /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/btree/bt_discard.c:137:9
          #4 0xffff8697c094 in __wt_ref_out /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/btree/bt_discard.c:43:5
          #5 0xffff86a7e56c in __wt_split_rewrite /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/btree/bt_split.c:2490:5
          #6 0xffff86ee0fcc in __evict_page_dirty_update /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_page.c:483:13
          #7 0xffff86edb670 in __wt_evict /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_page.c:300:9
          #8 0xffff86eb2b44 in __evict_page /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_lru.c:3045:5
          #9 0xffff86eb8210 in __evict_lru_pages /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_lru.c:1418:20
          #10 0xffff86eaa740 in __evict_thread_run /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_lru.c:355:9
          #11 0xffff87358b28 in __thread_run /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/support/thread_group.c:32:9
          #12 0xffff87b0579c in asan_thread_start(void*) /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:239:28
          #13 0x39ffff8652acd8  (<unknown module>)
      previously allocated by thread T119 here:
          #0 0xffff87b08124 in calloc /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_malloc_linux.cpp:75:3
          #1 0xffff86ff0170 in __wt_calloc /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/os_common/os_alloc.c:68:14
          #2 0xffff86b67074 in __wt_row_ikey_alloc /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/btree/row_key.c:375:5
          #3 0xffff8711d334 in __rec_split_write /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/reconcile/rec_write.c:2405:9
          #4 0xffff8711910c in __wti_rec_split_finish /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/reconcile/rec_write.c:1683:13
          #5 0xffff870bf71c in __wti_rec_row_leaf /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/reconcile/rec_row.c:1400:11
          #6 0xffff87111490 in __reconcile /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/reconcile/rec_write.c:310:9
          #7 0xffff8710c8a8 in __wt_reconcile /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/reconcile/rec_write.c:128:11
          #8 0xffff86edecd8 in __evict_reconcile /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_page.c:1076:9
          #9 0xffff86edac40 in __wt_evict /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_page.c:241:9
          #10 0xffff86eb2b44 in __evict_page /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_lru.c:3045:5
          #11 0xffff86eb8210 in __evict_lru_pages /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_lru.c:1418:20
          #12 0xffff86eaa740 in __evict_thread_run /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_lru.c:355:9
          #13 0xffff87358b28 in __thread_run /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/support/thread_group.c:32:9
          #14 0xffff87b0579c in asan_thread_start(void*) /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:239:28
          #15 0x39ffff8652acd8  (<unknown module>)
      Thread T119 created by T0 here:
          #0 0xffff87aecc34 in pthread_create /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_interceptors.cpp:250:3
          #1 0xffff8701b318 in __wt_thread_create /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/os_posix/os_thread.c:60:5
          #2 0xffff87355088 in __thread_group_resize /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/support/thread_group.c:209:9
          #3 0xffff87355d4c in __wt_thread_group_create /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/support/thread_group.c:295:5
          #4 0xffff86ea9d98 in __wt_evict_threads_create /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/evict/evict_lru.c:600:5
          #5 0xffff86ca5aac in __wti_connection_workers /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_open.c:299:5
          #6 0xffff86c1ab6c in wiredtiger_open /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_api.c:3501:5
          #7 0xaaaae7a523e0 in wts_open /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/test/format/wts.c:828:9
          #8 0xaaaae7a53128 in wts_reopen /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/test/format/wts.c:879:5
          #9 0xaaaae7a029f4 in disagg_switch_roles /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/test/format/format_disagg.c:157:9
          #10 0xaaaae7a3be88 in main /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/test/format/t.c:408:13
          #11 0xffff86462554 in __libc_start_call_main (/lib64/libc.so.6+0x30554) (BuildId: 7f46fcd5f30f2f31235144f8192c97c839732436)
          #12 0x7effff86462638  (<unknown module>)
          #13 0x3baaaae79df2ec  (<unknown module>)
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      SUMMARY: AddressSanitizer: heap-use-after-free /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_layered.c:1964:13 in __layered_drain_ingest_tables
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      Shadow bytes around the buggy address:
        0x503001a60a00: fa fa 00 00 01 fa fa fa 00 00 01 fa fa fa fa fa
        0x503001a60a80: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x503001a60b00: 00 00 01 fa fa fa fa fa fa fa fa fa fd fd fd fd
        0x503001a60b80: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x503001a60c00: fa fa fa fa 00 00 00 00 fa fa fa fa fa fa fa fa
      =>0x503001a60c80: fd fd[fd]fa fa fa fd fd fd fa fa fa 00 00 01 fa
        0x503001a60d00: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fd fd
        0x503001a60d80: fd fd fa fa fa fa fa fa fa fa 00 00 03 fa fa fa
        0x503001a60e00: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x503001a60e80: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
        0x503001a60f00: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
      Shadow byte legend (one shadow byte represents 8 application bytes):
        Addressable:           00
        Partially addressable: 01 02 03 04 05 06 07
        Heap left redzone:       fa
        Freed heap region:       fd
        Stack left redzone:      f1
        Stack mid redzone:       f2
        Stack right redzone:     f3
        Stack after return:      f5
        Stack use after scope:   f8
        Global redzone:          f9
        Global init order:       f6
        Poisoned by user:        f7
        Container overflow:      fc
        Array cookie:            ac
        Intra object redzone:    bb
        ASan internal:           fe
        Left alloca redzone:     ca
        Right alloca redzone:    cb
      

      logs

      format-stress-test-disagg-switch-1 task_log

      Logs:

      #0  0x0000ffff864bf7b4 in __pthread_kill_implementation () from /lib64/libc.so.6
      #0  0x0000ffff864bf7b4 in __pthread_kill_implementation () from /lib64/libc.so.6
      #1  0x0000ffff864763a0 [PAC] in raise () from /lib64/libc.so.6
      #2  0x0000ffff86462264 [PAC] in abort () from /lib64/libc.so.6
      #3  0x0000ffff87a5133c [PAC] in Abort () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp:163
      #4  0x0000ffff87a4f224 in __sanitizer::Die() () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/sanitizer_common/sanitizer_termination.cpp:58
      #5  0x0000ffff87b0cb80 in ~ScopedInErrorReport () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_report.cpp:193
      #6  0x0000ffff87b0fec0 in ReportGenericError () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_report.cpp:498
      #7  0x0000ffff87b10dd0 in __asan_report_load8 () at /data/mci/01bb46477e468e9b17d7d0a0c518db71/toolchain-builder/tmp/build-llvm-v5.sh-42U/llvm-project-llvmorg/compiler-rt/lib/asan/asan_rtl.cpp:131
      #8  0x0000ffff86c965c4 in __layered_drain_ingest_tables (session=0xffff8589e800) at /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_layered.c:1964
      #9  0x0000ffff86c87518 in __disagg_step_up (session=0xffff8589e800) at /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_layered.c:1206
      #10 0x0000ffff86c83c00 in __wti_disagg_conn_config (session=0xffff8589e800, cfg=0xffff848a4020, reconfig=true) at /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_layered.c:1306
      #11 0x0000ffff86cb6548 in __wti_conn_reconfig (session=0xffff8589e800, cfg=0xffff848a4020) at /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_reconfig.c:450
      #12 0x0000ffff86c228b8 in __conn_reconfigure (wt_conn=0x525000550100, config=0xffff843ab420 "disaggregated=(role=\"leader\")") at /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/src/conn/conn_api.c:1354
      #13 0x0000aaaae7a02ba0 in disagg_switch_roles () at /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/test/format/format_disagg.c:163
      #14 0x0000aaaae7a3be8c in main (argc=5, argv=0xffffdd195860) at /data/mci/aba579aa8eedf925df53be101b695a14/wiredtiger/test/format/t.c:408
      

      logs

      Repro Artifacts:

            Assignee:
            [DO NOT USE] Backlog - Storage Engines Team
            Reporter:
            xgen-buildbaron-user
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: