test/format (disagg.mode=follower) timeout

XMLWordPrintableJSON

      format-stress-data-validation-test-disagg-follower-1 on ubuntu2004-arm64-release-nonstandalone

      Host: i-0dbde5ea3a9871316
      Project: wiredtiger
      Commit: ce2200f3
      Please refer to BF(G) Playbook for instructions on handling BF and BFG tickets as well as Auto-Resolution Rules

      Task Logs:

      format-stress-data-validation-test-disagg-follower-1 task_log

      Logs:

      format run more than 15 minutes past the maximum time
      

      logs

      format-stress-data-validation-test-disagg-follower-1 task_log

      Logs:

      format run dumping cache and transaction state, then aborting the process
      
      

      logs

      format-stress-data-validation-test-disagg-follower-1 task_log

      Logs:

      0x7203bfc86000:transaction state dump
      0x7203bfc86000:current ID: 4786091
      0x7203bfc86000:last running ID: 4786091
      0x7203bfc86000:metadata_pinned ID: 4786091
      0x7203bfc86000:oldest ID: 4786091
      0x7203bfc86000:durable timestamp: (0, 40423731)
      0x7203bfc86000:oldest timestamp: (0, 40423615)
      0x7203bfc86000:pinned timestamp: (0, 40423615)
      0x7203bfc86000:stable timestamp: (0, 40423615)
      0x7203bfc86000:has_durable_timestamp: yes
      0x7203bfc86000:has_oldest_timestamp: yes
      0x7203bfc86000:has_pinned_timestamp: yes
      0x7203bfc86000:has_stable_timestamp: yes
      0x7203bfc86000:oldest_is_pinned: yes
      0x7203bfc86000:stable_is_pinned: yes
      0x7203bfc86000:checkpoint running: no
      0x7203bfc86000:checkpoint generation: 154
      0x7203bfc86000:checkpoint pinned ID: 0
      0x7203bfc86000:checkpoint txn ID: 0
      0x7203bfc86000:session count: 46
      0x7203bfc86000:Transaction state of active sessions:
      0x7203bfc86000:=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
      0x7203bfc86000:cache dump
      0x7203bfc86000:cache full: no
      0x7203bfc86000:cache clean check: yes (96.417%)
      0x7203bfc86000:cache dirty check: no (70.357%)
      0x7203bfc86000:cache updates check: no (74.593%)
      0x7203bfc86000:file:T00003.wt_ingest(<live>):
      ======= FAILURE ==========
      ############################################
      #  RUN PARAMETERS: V3
      ############################################
      assert.read_timestamp=0
      background_compact=0
      background_compact.free_space_target=87
      backup=0
      backup.incremental=off
      backup.incr_granularity=847
      backup.live_restore=0
      backup.live_restore_read_size=512
      backup.live_restore_threads=7
      block_cache=0
      block_cache.cache_on_checkpoint=0
      block_cache.cache_on_writes=1
      block_cache.size=28
      cache=5947
      cache.evict_max=5
      cache.eviction_dirty_target=0
      cache.eviction_dirty_trigger=95
      cache.eviction_updates_target=0
      cache.eviction_updates_trigger=95
      cache.minimum=0
      cache.maximum=0
      checkpoint=on
      checkpoint.log_size=120
      checkpoint.wait=15
      chunk_cache=0
      chunk_cache.capacity=1619
      chunk_cache.chunk_size=4
      chunk_cache.storage_path=off
      chunk_cache.type=off
      compact.free_space_target=35
      debug.background_compact=0
      debug.checkpoint_retention=4
      debug.cursor_reposition=0
      debug.eviction=0
      debug.log_retention=8
      debug.realloc_exact=0
      debug.realloc_malloc=0
      debug.slow_checkpoint=0
      debug.table_logging=0
      debug.update_restore_evict=0
      disagg.internal_page_delta=1
      disagg.leaf_page_delta=1
      disagg.multi=0
      disagg.enabled=1
      disagg.layered=1
      disagg.mode=follower
      disagg.page_log=palite
      disagg.page_log.verbose=0
      disk.data_extend=0
      disk.encryption=none
      disk.mmap=1
      disk.mmap_all=1
      eviction.evict_use_softptr=0
      file_manager.close_handle_minimum=16
      file_manager.close_idle_time=21
      file_manager.close_scan_interval=30
      format.abort=0
      format.independent_thread_rng=1
      format.major_timeout=0
      import=0
      logging=0
      logging.compression=none
      logging.file_max=320265
      logging.prealloc=1
      logging.remove=1
      obsolete_cleanup.method=off
      obsolete_cleanup.wait=3240
      ops.alter=0
      ops.compaction=0
      ops.hs_cursor=1
      ops.pct.modify=0
      ops.bound_cursor=0
      ops.prepare=0
      ops.random_cursor=1
      ops.salvage=0
      ops.throttle=0
      ops.throttle.sleep_us=578111
      ops.truncate=0
      ops.verify=1
      prefetch=0
      precise_checkpoint=1
      preserve_prepared=0
      quiet=1
      random.data_seed=14288327
      random.extra_seed=14626658
      runs.in_memory=0
      runs.mirror=1
      runs.ops=0
      runs.predictable_replay=0
      runs.source=layered
      runs.tables=3
      runs.threads=28
      runs.timer=13
      runs.type=row-store
      runs.verify_failure_dump=0
      statistics.mode=fast
      statistics_log.sources=off
      stress.aggressive_stash_free=0
      stress.aggressive_sweep=0
      stress.checkpoint=0
      stress.checkpoint_evict_page=0
      stress.checkpoint_prepare=0
      stress.compact_slow=0
      stress.evict_reposition=0
      stress.failpoint_eviction_split=0
      stress.failpoint_hs_delete_key_from_ts=0
      stress.failpoint_rec_before_wrapup=0
      stress.hs_checkpoint_delay=0
      stress.hs_search=0
      stress.hs_sweep=0
      stress.prefetch_delay=0
      stress.prepare_resolution_1=0
      stress.sleep_before_read_overflow_onpage=0
      stress.split_1=0
      stress.split_2=0
      stress.split_3=0
      stress.split_4=0
      stress.split_5=0
      stress.split_6=0
      stress.split_7=0
      stress.split_8=0
      tiered_storage.flush_frequency=0
      tiered_storage.storage_source=off
      transaction.implicit=0
      transaction.operation_timeout_ms=2000
      transaction.timestamps=1
      wiredtiger.config=off
      wiredtiger.rwlock=0
      wiredtiger.leak_memory=0
      ############################################
      #  TABLE PARAMETERS: table 1
      ############################################
      table1.btree.compression=none
      table1.btree.dictionary=0
      table1.btree.internal_key_truncation=1
      table1.btree.internal_page_max=17
      table1.btree.key_max=89
      table1.btree.key_min=27
      table1.btree.leaf_page_max=9
      table1.btree.memory_page_max=2
      table1.btree.prefix_len=0
      table1.btree.prefix_compression=0
      table1.btree.prefix_compression_min=3
      table1.btree.reverse=0
      table1.btree.split_pct=98
      table1.btree.value_max=3302
      table1.btree.value_min=1
      table1.disk.checksum=uncompressed
      table1.disk.firstfit=0
      table1.ops.pareto=0
      table1.ops.pareto.skew=29
      table1.ops.pct.delete=8
      table1.ops.pct.insert=91
      table1.ops.pct.read=0
      table1.ops.pct.write=1
      table1.runs.rows=119960
      ############################################
      #  TABLE PARAMETERS: table 2
      ############################################
      table2.btree.compression=none
      table2.btree.dictionary=1
      table2.btree.internal_key_truncation=0
      table2.btree.internal_page_max=12
      table2.btree.key_max=122
      table2.btree.key_min=13
      table2.btree.leaf_page_max=17
      table2.btree.memory_page_max=6
      table2.btree.prefix_len=0
      table2.btree.prefix_compression=1
      table2.btree.prefix_compression_min=5
      table2.btree.reverse=0
      table2.btree.split_pct=96
      table2.btree.value_max=3095
      table2.btree.value_min=17
      table2.disk.checksum=unencrypted
      table2.disk.firstfit=0
      table2.ops.pareto=0
      table2.ops.pareto.skew=72
      table2.ops.pct.delete=54
      table2.ops.pct.insert=14
      table2.ops.pct.read=23
      table2.ops.pct.write=9
      table2.runs.rows=119960
      ############################################
      #  TABLE PARAMETERS: table 3
      ############################################
      table3.btree.compression=none
      table3.btree.dictionary=0
      table3.btree.internal_key_truncation=1
      table3.btree.internal_page_max=13
      table3.btree.key_max=124
      table3.btree.key_min=15
      table3.btree.leaf_page_max=10
      table3.btree.memory_page_max=3
      table3.btree.prefix_len=0
      table3.btree.prefix_compression=1
      table3.btree.prefix_compression_min=0
      table3.btree.reverse=0
      table3.btree.split_pct=82
      table3.btree.value_max=1933
      table3.btree.value_min=6
      table3.disk.checksum=unencrypted
      table3.disk.firstfit=1
      table3.ops.pareto=0
      table3.ops.pareto.skew=93
      table3.ops.pct.delete=15
      table3.ops.pct.insert=7
      table3.ops.pct.read=72
      table3.ops.pct.write=6
      table3.runs.rows=119960
      bash: line 55: 11409 Segmentation fault      (core dumped) ./t -R $format_args
      Command 'shell.exec' in function 'format test disagg' (step 2 of 2) failed: shell script encountered problem: exit code 1.
      Finished command 'shell.exec' in function 'format test disagg' (step 2 of 2) in 3h25m38.691018254s.
      Running task commands failed: running command: command failed: shell script encountered problem: exit code 1
      Finished running task commands in 3h25m45.666241231s.
      Task completed - FAILURE.
      Running post-task commands.
      Running command 'shell.exec' in function 'dump stacktraces' (step 1 of 8) in block 'post'.
      +---------------------------------+
      | ./test/format/dump_t.11409.core |
      +---------------------------------+
      [New LWP 11409]
      [New LWP 11410]
      [New LWP 11411]
      [New LWP 11412]
      [New LWP 11413]
      [New LWP 11424]
      python3 ../test/evergreen/print_stack_trace.py
      [New LWP 11451]
      warning: Could not load shared library symbols for 7 libraries, e.g. ../..//ext/page_log/palite/libwiredtiger_palite.so.
      [New LWP 11417]
      [New LWP 11442]
      Use the "info sharedlibrary" command to see the complete listing.
      [New LWP 11416]
      Do you need "set solib-search-path" or "set sysroot"?
      [New LWP 11414]
      [New LWP 11435]
      [New LWP 11418]
      [New LWP 11419]
      [New LWP 11450]
      [New LWP 11447]
      [New LWP 11415]
      [New LWP 11427]
      [New LWP 11441]
      [New LWP 11428]
      [New LWP 11437]
      [New LWP 11429]
      [New LWP 11440]
      [New LWP 11443]
      [New LWP 11446]
      [New LWP 11445]
      [New LWP 11431]
      [New LWP 11449]
      [New LWP 11444]
      [New LWP 11448]
      [New LWP 11432]
      [New LWP 11434]
      [New LWP 11438]
      [New LWP 11439]
      [New LWP 11436]
      [New LWP 11420]
      [New LWP 11430]
      [New LWP 11421]
      [New LWP 11433]
      [New LWP 11426]
      [New LWP 11422]
      [New LWP 11423]
      [New LWP 11425]
      [Thread debugging using libthread_db enabled]
      Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
      Core was generated by `./t -R'.
      Program terminated with signal SIGSEGV, Segmentation fault.
      #0  0x0000ffff94b61a54 in __wt_page_release (session=session@entry=0x7203bfc86000, ref=ref@entry=0x72039612fa80, flags=flags@entry=9267) at /data/mci/8ec15a3de71dca7b9616555752ffd3b5/wiredtiger/src/include/ref_inline.h:18
      18	/data/mci/8ec15a3de71dca7b9616555752ffd3b5/wiredtiger/src/include/ref_inline.h: No such file or directory.
      [Current thread is 1 (Thread 0xffff95efc040 (LWP 11409))]
      Loading custom WiredTiger gdb scripts...
      #0  0x0000ffff94b61a54 in __wt_page_release (session=session@entry=0x7203bfc86000, ref=ref@entry=0x72039612fa80, flags=flags@entry=9267) at /data/mci/8ec15a3de71dca7b9616555752ffd3b5/wiredtiger/src/include/ref_inline.h:18
      #1  0x0000ffff94b62d3c in __tree_walk_internal (session=session@entry=0x7203bfc86000, refp=0xffffdcb3e3f8, walkcntp=walkcntp@entry=0x0, skip_func=skip_func@entry=0x0, func_cookie=func_cookie@entry=0x0, flags=9267, flags@entry=8243) at /data/mci/8ec15a3de71dca7b9616555752ffd3b5/wiredtiger/src/btree/bt_walk.c:403
      #2  0x0000ffff94b63234 in __wt_tree_walk (session=session@entry=0x7203bfc86000, refp=<optimized out>, flags=flags@entry=8243) at /data/mci/8ec15a3de71dca7b9616555752ffd3b5/wiredtiger/src/btree/bt_walk.c:415
      #3  0x0000ffff94acb498 in __verbose_dump_cache_single (session=0x7203bfc86000, total_bytesp=0xffffdcb3e458, total_dirty_bytesp=0xffffdcb3e460, total_updates_bytesp=0xffffdcb3e468) at /data/mci/8ec15a3de71dca7b9616555752ffd3b5/wiredtiger/src/evict/evict_lru.c:3337
      #4  __verbose_dump_cache_apply (session=session@entry=0x7203bfc86000, total_bytesp=total_
      

      logs

      Repro Artifacts:

            Assignee:
            Sid Mahajan
            Reporter:
            xgen-buildbaron-user
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

              Created:
              Updated:
              Resolved: