Triage validation failures found in disagg

XMLWordPrintableJSON

    • Type: Task
    • Resolution: Unresolved
    • Priority: Major - P3
    • None
    • Affects Version/s: None
    • Component/s: Verify
    • None
    • Storage Engines, Storage Engines - Persistence, Storage Engines - Transactions
    • SE Persistence - 2025-10-24
    • None

      Restore/ startup successful, then hit validation error.

      {"t":{"$date":"2025-10-19T01:07:39.974+00:00"},"s":"I",  "c":"COMMAND",  "id":20514,   "ctx":"conn97","msg":"CMD: validate","attr":{"namespace":"internal_beta_workloads.ttl_test_collection","background":false,"full":true,"extended":false,"enforc
      eFastCount":false,"checkBSONConformance":true,"fixMultiKey":false,"repair":false,"wiredtigerVerifyConfigurationOverride":null}}
      {"t":{"$date":"2025-10-19T01:07:40.017+00:00"},"s":"W",  "c":"WT",       "id":22430,   "ctx":"conn97","msg":"WiredTiger message","attr":{"message":{"ts_sec":1760836060,"ts_usec":17773,"thread":"438573:0xffff80437200","session_dhandle_name":"file
      :collection-d85457a3-1479-4ba1-aef5-f1f7fa60aebd.wt_stable","session_name":"WT_SESSION.verify","category":"WT_VERB_DISAGGREGATED_STORAGE","log_id":1000000,"category_id":13,"verbose_level":"WARNING","verbose_level_id":-2,"msg":"plh_discard is not
       implemented"}}}
      {"t":{"$date":"2025-10-19T01:07:40.018+00:00"},"s":"I",  "c":"INDEX",    "id":20295,   "ctx":"conn97","msg":"Validating internal structure","attr":{"index":"_id_","namespace":"internal_beta_workloads.ttl_test_collection"}}
      {"t":{"$date":"2025-10-19T01:07:40.034+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"thread170","msg":"WiredTiger error message","attr":{"error":0,"message":{"ts_sec":1760836060,"ts_usec":34703,"thread":"438573:0xffffa4827200","session_
      dhandle_name":"file:index-07295dd9-763d-4f9c-b8da-db16f00ecdb2.wt_stable","session_name":"prefetch-server","category":"WT_VERB_DEFAULT","log_id":1000000,"category_id":12,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"void __wt_cache_decr_c
      heck_uint64(WT_SESSION_IMPL *, uint64_t *, uint64_t, const char *):459:dirty leaf page count was 0, went negative with decrement of 1"}}}
      {"t":{"$date":"2025-10-19T01:07:40.036+00:00"},"s":"E",  "c":"WT",       "id":22436,   "ctx":"conn97","msg":"WiredTiger error message","attr":{"error":22,"message":{"ts_sec":1760836060,"ts_usec":36353,"thread":"438573:0xffff80437200","session_dh
      andle_name":"file:index-07295dd9-763d-4f9c-b8da-db16f00ecdb2.wt_stable","session_name":"WT_SESSION.verify","category":"WT_VERB_DEFAULT","log_id":1000000,"category_id":12,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __time_value_valid
      ate_parent(WT_SESSION_IMPL *, WT_TIME_WINDOW *, WT_TIME_AGGREGATE *, _Bool):466:value time window has a stop time after its parent's newest stop time; time window start: (0, 0)/(0, 0)/(0, 0)/0/0 | stop: (0, 0)/(4294967295, 4294967295)/(0, 0)/0/1
      8446744073709551605, parent newest_durable: (0, 0)/(1760830944, 234) | oldest_start: (0, 0)/0 | newest_stop: (1760830944, 234)/0","error_str":"Invalid argument","error_code":22}}}
      {"t":{"$date":"2025-10-19T01:07:40.036+00:00"},"s":"E",  "c":"WT",       "id":22436,   "ctx":"conn97","msg":"WiredTiger error message","attr":{"error":22,"message":{"ts_sec":1760836060,"ts_usec":36425,"thread":"438573:0xffff80437200","session_dhandle_name":"file:index-07295dd9-763d-4f9c-b8da-db16f00ecdb2.wt_stable","session_name":"WT_SESSION.verify","category":"WT_VERB_DEFAULT","log_id":1000000,"category_id":12,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __verify_page_content_leaf(WT_SESSION_IMPL *, WT_REF *, WT_CELL_UNPACK_ADDR *, WT_VSTUFF *):1358:cell 499 on page at [146, 1, 7562711322559774726, 7562711064861736968, 590, 712030481] newest_durable: (0, 0)/(1760830944, 234) | oldest_start: (0, 0)/0 | newest_stop: (1760830944, 234)/0 failed timestamp validation","error_str":"Invalid argument","error_code":22}}}
      {"t":{"$date":"2025-10-19T01:07:40.039+00:00"},"s":"E",  "c":"WT",       "id":22436,   "ctx":"conn97","msg":"WiredTiger error message","attr":{"error":22,"message":{"ts_sec":1760836060,"ts_usec":39201,"thread":"438573:0xffff80437200","session_dhandle_name":"layered:index-07295dd9-763d-4f9c-b8da-db16f00ecdb2","session_name":"WT_SESSION.verify","category":"WT_VERB_DEFAULT","log_id":1000000,"category_id":12,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"int __schema_layered_worker_verify(WT_SESSION_IMPL *, const char *, int (*)(WT_SESSION_IMPL *, const char **), int (*)(WT_SESSION_IMPL *, const char *, _Bool *), const char **, uint32_t):110:Verify (layered): file:index-07295dd9-763d-4f9c-b8da-db16f00ecdb2.wt_stable stable table verification failed ","error_str":"Invalid argument","error_code":22}}}
      {"t":{"$date":"2025-10-19T01:07:40.039+00:00"},"s":"E",  "c":"STORAGE",  "id":51782,   "ctx":"conn97","msg":"verify() returned an error. This indicates structural damage. Not examining individual index entries.","attr":{"error":"Invalid argument"}}
      {"t":{"$date":"2025-10-19T01:07:40.039+00:00"},"s":"I",  "c":"INDEX",    "id":20295,   "ctx":"conn97","msg":"Validating internal structure","attr":{"index":"expireAt_1","namespace":"internal_beta_workloads.ttl_test_collection"}}
      {"t":{"$date":"2025-10-19T01:07:40.089+00:00"},"s":"I",  "c":"WT",       "id":22431,   "ctx":"conn97","msg":"WiredTiger progress","attr":{"operation":"WT_SESSION.verify","progress":100}}
      {"t":{"$date":"2025-10-19T01:07:40.089+00:00"},"s":"W",  "c":"WT",       "id":22430,   "ctx":"conn97","msg":"WiredTiger message","attr":{"message":{"ts_sec":1760836060,"ts_usec":89749,"thread":"438573:0xffff80437200","session_dhandle_name":"file:index-0d16aece-a00e-41bc-a254-4250385b06d0.wt_stable","session_name":"WT_SESSION.verify","category":"WT_VERB_DISAGGREGATED_STORAGE","log_id":1000000,"category_id":13,"verbose_level":"WARNING","verbose_level_id":-2,"msg":"plh_discard is not implemented"}}}
      {"t":{"$date":"2025-10-19T01:07:40.090+00:00"},"s":"E",  "c":"STORAGE",  "id":7463100, "ctx":"conn97","msg":"Index failed validation","attr":{"spec":{"v":2,"key":{"_id":1},"name":"_id_"}}}
      {"t":{"$date":"2025-10-19T01:07:40.090+00:00"},"s":"I",  "c":"INDEX",    "id":20302,   "ctx":"conn97","msg":"Validation complete -- Corruption found","attr":{"namespace":"internal_beta_workloads.ttl_test_collection","uuid":{"uuid":{"$uuid":"348a5150-5dda-475f-93f9-45341cc0c84e"}}}}
      

            Assignee:
            Etienne Petrel
            Reporter:
            Peter Vertenten
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

              Created:
              Updated: