disagg: checkpoint returned errno 2

XMLWordPrintableJSON

    • Storage Engines - Foundations
    • 1,891.953
    • SE Persistence - 2025-11-07, SE Persistence - 2025-11-21, SE Persistence - 2026-02-27
    • 8

      relevant log snippet:

      {"t":{"$date":"2025-10-14T17:20:53.854+00:00"},"s":"I",  "c":"WTCHKPT",  "id":22430,   "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1760462453,"ts_usec":854478,"thread":"286630:0xffff7b608200","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT","log_id":1000000,"category_id":5,"verbose_level":"INFO","verbose_le
      vel_id":0,"msg":"Checkpoint requested at stable timestamp (1760462452, 1)"}}}
      {"t":{"$date":"2025-10-14T17:20:53.856+00:00"},"s":"D3", "c":"DISAGG",   "id":10985400,"ctx":"Disagg-0","msg":"LogServerManager::_standbyLoop read","attr":{"readRequest":"filters { log_segment_id { log_id: 1845250389136819 } start_lsn { lsn: 7560985720664358913 } entry_type_filters: 0 entry_type_filters: 2 }"}}
      {"t":{"$date":"2025-10-14T17:20:53.856+00:00"},"s":"D3", "c":"DISAGG",   "id":10985335,"ctx":"Disagg-0","msg":"Received new materialized offsets","attr":{"offsets":"min_recovery_lsn { lsn: 7561128657175969793 } log_upload_lsn { lsn: 7561128584161525820 } log_upload_lsn { domain_id: 1 lsn: 7561128588456493248 } log_materialized_lsn { domain_id: 1 lsn: 756112866
      1470937207 }","prevMaterializedLsn":7561128661470937207}}
      {"t":{"$date":"2025-10-14T17:20:53.858+00:00"},"s":"D3", "c":"DISAGG",   "id":10985400,"ctx":"Disagg-0","msg":"LogServerManager::_standbyLoop read","attr":{"readRequest":"filters { log_segment_id { log_id: 1845250389136819 } start_lsn { lsn: 7560985720664358913 } entry_type_filters: 0 entry_type_filters: 2 }"}}
      {"t":{"$date":"2025-10-14T17:20:53.858+00:00"},"s":"D3", "c":"DISAGG",   "id":10985335,"ctx":"Disagg-0","msg":"Received new materialized offsets","attr":{"offsets":"min_recovery_lsn { lsn: 7561128657175969793 } log_upload_lsn { lsn: 7561128584161525820 } log_upload_lsn { domain_id: 1 lsn: 7561128588456493248 } log_materialized_lsn { domain_id: 1 lsn: 756112866
      1470937225 }","prevMaterializedLsn":7561128661470937207}}
      {"t":{"$date":"2025-10-14T17:20:53.858+00:00"},"s":"D3", "c":"DISAGG",   "id":10985340,"ctx":"Disagg-0","msg":"Updating materialized LSN","attr":{"newMaterializedLsn":7561128661470937225,"prevMaterializedLsn":7561128661470937207}}
      {"t":{"$date":"2025-10-14T17:20:53.858+00:00"},"s":"D3", "c":"DISAGG",   "id":10985330,"ctx":"Disagg-0","msg":"CheckpointManager::_updateCheckpointIfAvailable","attr":{"latestMaterializedAndAppliedCheckpointLsn":7561128661470937225,"_installedCheckpointLsn":7561128588456493248,"_checkpointToInstallLsn":0}}
      {"t":{"$date":"2025-10-14T17:20:53.858+00:00"},"s":"D3", "c":"EXECUTOR", "id":23109,   "ctx":"Disagg-137","msg":"Executing a task on behalf of pool","attr":{"poolName":"DisaggThreadPool"}}
      {"t":{"$date":"2025-10-14T17:20:53.858+00:00"},"s":"D2", "c":"DISAGG",   "id":10985338,"ctx":"Disagg-137","msg":"Setting storage checkpoint","attr":{"checkpointLsn":7561128661470937225,"checkpointStableTimestamp":{"$timestamp":{"t":1760462452,"i":1}},"checkpointMetadata":"metadata_lsn=7561128661470937224"}}
      {"t":{"$date":"2025-10-14T17:20:53.868+00:00"},"s":"F",  "c":"ASSERT",   "id":23083,   "ctx":"Checkpointer","msg":"Invariant failure","attr":{"expr":"wtRet","error":"NoSuchKey: 2: No such file or directory - ","location":"src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp:2143:33:void mongo::WiredTigerKVEngine::_checkpoint(WiredTigerSession &, bool)"}}
      {"t":{"$date":"2025-10-14T17:20:53.868+00:00"},"s":"F",  "c":"ASSERT",   "id":23084,   "ctx":"Checkpointer","msg":"\n\n***aborting after invariant() failure\n\n"}
      

      Could we add more WT messages when we're about to return "2" from checkpoint?

        1. Archive.zip
          37.46 MB
        2. error.json
          1.37 MB
        3. extractor.py
          1 kB
        4. missing-table.nb2
          2 kB
        5. mongod_2.core.gz
          62.95 MB
        6. wt-15766-repro-1.nb2
          2 kB

            Assignee:
            Albert Song
            Reporter:
            Eric Milkie
            Votes:
            0 Vote for this issue
            Watchers:
            9 Start watching this issue

              Created:
              Updated:
              Resolved: