-
Type:
Build Failure
-
Resolution: Duplicate
-
Priority:
Major - P3
-
None
-
Affects Version/s: None
-
Component/s: Evergreen
format-stress-test-disagg-switch-1 on amazon2023-disagg-stress
Host: i-0bf41459fe5043d80
Project: wiredtiger
Commit: 55c43055
Please refer to BF(G) Playbook for instructions on handling BF and BFG tickets as well as Auto-Resolution Rules
Task Logs:
format-stress-test-disagg-switch-1 task_log
Logs:
[1757211575:804212][4382:0xffff2e7beb80], t, file:T00003.wt_stable, eviction-server: [WT_VERB_DEFAULT][ERROR]: __wti_block_disagg_write_internal, 118: WiredTiger assertion failed: 'conn->layered_table_manager.leader'. Trying to write the page from a follower [1757211575:804242][4382:0xffff2e7beb80], t, file:T00003.wt_stable, eviction-server: [WT_VERB_DEFAULT][ERROR]: __wt_abort, 29: aborting WiredTiger library
format-stress-test-disagg-switch-1 task_log
Logs:
#0 0x0000ffffb240e9b4 in __pthread_kill_implementation () from /lib64/libc.so.6
#0 0x0000ffffb240e9b4 in __pthread_kill_implementation () from /lib64/libc.so.6
#1 0x0000ffffb23c53a0 [PAC] in raise () from /lib64/libc.so.6
#2 0x0000ffffb23b1264 [PAC] in abort () from /lib64/libc.so.6
#3 0x0000ffffb21c6a18 [PAC] in __wt_abort (session=session@entry=0x11bfffd0a880) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/os_common/os_abort.c:31
#4 0x0000ffffb20679e0 in __wti_block_disagg_write_internal (session=session@entry=0x11bfffd0a880, block_disagg=0x11bffa7f4e40, buf=buf@entry=0x11bfed38f780, block_meta=block_meta@entry=0x11bff812bc80, sizep=sizep@entry=0xffff2e7b9a50, checksump=checksump@entry=0xffff2e7b9a54, data_checksum=data_checksum@entry=true, checkpoint_io=checkpoint_io@entry=false) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/block_disagg/block_disagg_write.c:118
#5 0x0000ffffb2067af0 in __wti_block_disagg_write (session=session@entry=0x11bfffd0a880, block=<optimized out>, buf=buf@entry=0x11bfed38f780, block_meta=block_meta@entry=0x11bff812bc80, addr=addr@entry=0xffff2e7bad30 "P\367{.\377\377", addr_sizep=addr_sizep@entry=0xffff2e7bae38, data_checksum=data_checksum@entry=true, checkpoint_io=checkpoint_io@entry=false) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/block_disagg/block_disagg_write.c:243
#6 0x0000ffffb20655a0 in __bmd_write (bm=0x11bfeb2b8380, session=0x11bfffd0a880, buf=0x11bfed38f780, block_meta=0x11bff812bc80, addr=0xffff2e7bad30 "P\367{.\377\377", addr_sizep=0xffff2e7bae38, data_checksum=true, checkpoint_io=false) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/block_disagg/block_disagg_mgr.c:109
#7 0x0000ffffb2061104 in __wt_blkcache_write (session=session@entry=0x11bfffd0a880, buf=buf@entry=0x11bff1b00e08, block_meta=block_meta@entry=0x11bff812bc80, addr=addr@entry=0xffff2e7bad30 "P\367{.\377\377", addr_sizep=addr_sizep@entry=0xffff2e7bae38, compressed_sizep=compressed_sizep@entry=0xffff2e7bae30, checkpoint=checkpoint@entry=false, checkpoint_io=checkpoint_io@entry=false, compressed=compressed@entry=false) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/block_cache/block_io.c:789
#8 0x0000ffffb2200e18 in __rec_write (session=session@entry=0x11bfffd0a880, buf=buf@entry=0x11bff1b00e08, block_meta=0x11bff812bc80, addr=addr@entry=0xffff2e7bad30 "P\367{.\377\377", addr_sizep=addr_sizep@entry=0xffff2e7bae38, compressed_sizep=compressed_sizep@entry=0xffff2e7bae30, checkpoint=checkpoint@entry=false, checkpoint_io=false, compressed=compressed@entry=false) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_write.c:984
#9 0x0000ffffb220135c in __rec_write_image (session=session@entry=0x11bfffd0a880, r=r@entry=0x11bff1b00c80, chunk=chunk@entry=0x11bff1b00ce8, addr=addr@entry=0xffff2e7bad30 "P\367{.\377\377", addr_sizep=addr_sizep@entry=0xffff2e7bae38, compressed_sizep=compressed_sizep@entry=0xffff2e7bae30, last_block=last_block@entry=false) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_write.c:2635
#10 0x0000ffffb22088a0 in __rec_split_write (session=session@entry=0x11bfffd0a880, r=r@entry=0x11bff1b00c80, chunk=0x11bff1b00ce8, last_block=last_block@entry=false) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_write.c:2889
#11 0x0000ffffb2208b14 in __wti_rec_split (session=session@entry=0x11bfffd0a880, r=r@entry=0x11bff1b00c80, next_len=next_len@entry=70) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_write.c:1586
#12 0x0000ffffb2208eac in __wti_rec_split_crossing_bnd (session=session@entry=0x11bfffd0a880, r=r@entry=0x11bff1b00c80, next_len=70) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_write.c:1693
#13 0x0000ffffb21f2500 in __rec_row_leaf_insert (session=session@entry=0x11bfffd0a880, r=r@entry=0x11bff1b00c80, ins=0x11bff30d3b80) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_row.c:847
#14 0x0000ffffb21f4844 in __wti_rec_row_leaf (session=session@entry=0x11bfffd0a880, r=r@entry=0x11bff1b00c80, pageref=pageref@entry=0x11bfeb33df40, salvage=salvage@entry=0x0) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_row.c:1263
#15 0x0000ffffb2206eb8 in __reconcile (session=session@entry=0x11bfffd0a880, ref=ref@entry=0x11bfeb33df40, salvage=salvage@entry=0x0, flags=flags@entry=1184, page_lockedp=page_lockedp@entry=0xffff2e7bcb0f) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_write.c:305
#16 0x0000ffffb22061b0 in __wt_reconcile (session=session@entry=0x11bfffd0a880, ref=ref@entry=0x11bfeb33df40, salvage=salvage@entry=0x0, flags=flags@entry=1184) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/reconcile/rec_write.c:124
#17 0x0000ffffb2192f9c in __evict_reconcile (session=session@entry=0x11bfffd0a880, ref=ref@entry=0x11bfeb33df40, evict_flags=evict_flags@entry=0) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/evict/evict_page.c:1055
#18 0x0000ffffb219500c in __wt_evict (session=session@entry=0x11bfffd0a880, ref=0x11bfeb33df40, previous_state=3 '\003', flags=flags@entry=0) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/evict/evict_page.c:236
#19 0x0000ffffb218a4a8 in __evict_page (session=session@entry=0x11bfffd0a880, is_server=is_server@entry=true) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/evict/evict_lru.c:2958
#20 0x0000ffffb218a954 in __evict_lru_pages (session=session@entry=0x11bfffd0a880, is_server=is_server@entry=true) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/evict/evict_lru.c:1407
#21 0x0000ffffb218e46c in __evict_pass (session=session@entry=0x11bfffd0a880) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/evict/evict_lru.c:920
#22 0x0000ffffb218e8a8 in __evict_server (session=session@entry=0x11bfffd0a880, did_work=did_work@entry=0xffff2e7bde5f) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/evict/evict_lru.c:462
#23 0x0000ffffb218ebbc in __evict_thread_run (session=0x11bfffd0a880, thread=0x11bfffe089b0) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/evict/evict_lru.c:330
#24 0x0000ffffb2265098 in __thread_run (arg=0x11bfffe089b0) at /data/mci/15c1bfa21e57855d156cbc74b8f78ce9/wiredtiger/src/support/thread_group.c:31
#25 0x0000ffffb240cd78 in start_thread () from /lib64/libc.so.6
#26 0x0000ffffb2479ddc [PAC] in thread_start () from /lib64/libc.so.6
Repro Artifacts: