__wti_page_inmem_updates assertion failure (disagg)

XMLWordPrintableJSON

    • Storage Engines, Storage Engines - Transactions
    • SE Transactions - 2025-10-10
    • None

      On startup, hitting an assertion failure (bunch of different functions hitting errors here)

      {"t":{"$date":"2025-10-01T15:56:50.082+00:00"},"s":"I",  "c":"STORAGE",  "id":9529901, "ctx":"Disagg-0","msg":"Initializing durable catalog","attr":{"numRecords":0}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Disagg-0","msg":"WiredTiger error message","attr":{"error":0,"message":{"ts_sec":1759334210,"ts_usec":85025,"thread":"498019:0xffff811f9300","session_dhandle_name":"file:_mdb_catalog.wt_stable/Wire
      dTigerCheckpoint.446","session_name":"WT_CURSOR.next","category":"WT_VERB_DEFAULT","log_id":1000000,"category_id":12,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __wti_page_inmem_updates(WT_SESSION_IMPL *, WT_REF *):981:WiredTiger assertion failed: '((((first_upd)->flags) & (0x0
      200u)) != 0)'. Expression returned false"}}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Disagg-0","msg":"WiredTiger error message","attr":{"error":0,"message":{"ts_sec":1759334210,"ts_usec":85091,"thread":"498019:0xffff811f9300","session_dhandle_name":"file:_mdb_catalog.wt_stable/Wire
      dTigerCheckpoint.446","session_name":"WT_CURSOR.next","category":"WT_VERB_DEFAULT","log_id":1000000,"category_id":12,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"void __wt_abort(WT_SESSION_IMPL *):29:aborting WiredTiger library"}}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Disagg-0","msg":"WiredTiger error message","attr":{"error":-31803,"message":{"ts_sec":1759334210,"ts_usec":85119,"thread":"498019:0xffff811f9300","session_dhandle_name":"file:_mdb_catalog.wt_stable
      /WiredTigerCheckpoint.446","session_name":"WT_CURSOR.next","category":"WT_VERB_ERROR_RETURNS","log_id":1000000,"category_id":14,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __wt_btcur_next(WT_CURSOR_BTREE *, _Bool):955:Error at src/btree/bt_curnext.c:955: \"WT_NOTFOUND\" failed"
      ,"error_str":"WT_NOTFOUND: item not found","error_code":-31803}}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Disagg-0","msg":"WiredTiger error message","attr":{"error":-31803,"message":{"ts_sec":1759334210,"ts_usec":85159,"thread":"498019:0xffff811f9300","session_dhandle_name":"file:_mdb_catalog.wt_stable
      /WiredTigerCheckpoint.446","session_name":"WT_CURSOR.next","category":"WT_VERB_ERROR_RETURNS","log_id":1000000,"category_id":14,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __curfile_next(WT_CURSOR *):186:Error at src/cursor/cur_file.c:186: \"ret\" failed","error_str":"WT_NOTFOU
      ND: item not found","error_code":-31803}}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Disagg-0","msg":"WiredTiger error message","attr":{"error":-31803,"message":{"ts_sec":1759334210,"ts_usec":85184,"thread":"498019:0xffff811f9300","session_dhandle_name":"file:_mdb_catalog.wt_stable
      /WiredTigerCheckpoint.446","session_name":"WT_CURSOR.next","category":"WT_VERB_ERROR_RETURNS","log_id":1000000,"category_id":14,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __wt_btcur_prev(WT_CURSOR_BTREE *, _Bool):898:Error at src/btree/bt_curprev.c:898: \"WT_NOTFOUND\" failed"
      ,"error_str":"WT_NOTFOUND: item not found","error_code":-31803}}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Disagg-0","msg":"WiredTiger error message","attr":{"error":-31803,"message":{"ts_sec":1759334210,"ts_usec":85207,"thread":"498019:0xffff811f9300","session_dhandle_name":"file:_mdb_catalog.wt_stable
      /WiredTigerCheckpoint.446","session_name":"WT_CURSOR.next","category":"WT_VERB_ERROR_RETURNS","log_id":1000000,"category_id":14,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __curfile_prev(WT_CURSOR *):246:Error at src/cursor/cur_file.c:246: \"ret\" failed","error_str":"WT_NOTFOU
      ND: item not found","error_code":-31803}}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Disagg-0","msg":"WiredTiger error message","attr":{"error":-31803,"message":{"ts_sec":1759334210,"ts_usec":85228,"thread":"498019:0xffff811f9300","session_dhandle_name":"file:_mdb_catalog.wt_stable
      /WiredTigerCheckpoint.446","session_name":"WT_CURSOR.next","category":"WT_VERB_ERROR_RETURNS","log_id":1000000,"category_id":14,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __curhs_search_near(WT_CURSOR *, int *):860:Error at src/cursor/cur_hs.c:860: \"ret\" failed","error_str":
      "WT_NOTFOUND: item not found","error_code":-31803}}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"Disagg-0","msg":"WiredTiger error message","attr":{"error":-31803,"message":{"ts_sec":1759334210,"ts_usec":85251,"thread":"498019:0xffff811f9300","session_dhandle_name":"file:_mdb_catalog.wt_stable
      /WiredTigerCheckpoint.446","session_name":"WT_CURSOR.next","category":"WT_VERB_ERROR_RETURNS","log_id":1000000,"category_id":14,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __curhs_search_near_helper(WT_SESSION_IMPL *, WT_CURSOR *, _Bool):732:Error at src/cursor/cur_hs.c:732: \"
      cursor->search_near(cursor, &cmp)\" failed","error_str":"WT_NOTFOUND: item not found","error_code":-31803}}}
      {"t":{"$date":"2025-10-01T15:56:50.085+00:00"},"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"Disagg-0","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
      

      This is from node 00-00. Node 00-01 has similar failures on startup.
      cluster: https://cloud-dev.mongodb.com/v2/68668c0a6274e169c105d235#/clusters/detail/sls-smoke-dev-aws-use1
      logs: https://cloud-dev.mongodb.com/log/collection/request/68668c0a6274e169c105d235/68dd4ff371c39f407ef0745e/download

            Assignee:
            Chenhao Qu
            Reporter:
            Eric Milkie
            Votes:
            0 Vote for this issue
            Watchers:
            7 Start watching this issue

              Created:
              Updated:
              Resolved: