Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-12084

Format testing hang - run more than 15 minutes

    • Type: Icon: Bug Bug
    • Resolution: Unresolved
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: None
    • Labels:
    • StorEng - Defined Pipeline

      Problem Description:

      While testing chunk cache, which is currently disabled in format testing (WT-10822) (when tiered testing is enabled, chunk cache testing will follow) I ran into a timeout issue.

      Acceptance criteria:

      The ticket requires the following work - 

      • Reproduce the test/format hangs locally. 
      • Investigate and fix the hangs.

      How to run the testing locally - 

      • For a fresh run (reproduces 1/10 times):
      ./t -c CONFIG 

      CONFIG - 

      ############################################
      #  RUN PARAMETERS: V3
      ############################################
      assert.read_timestamp=0
      background_compact=0
      background_compact.free_space_target=67
      backup=0
      backup.incremental=off
      backup.incr_granularity=1945
      block_cache=0
      block_cache.cache_on_checkpoint=0
      block_cache.cache_on_writes=1
      block_cache.size=40
      buffer_alignment=0
      cache=1476
      cache.evict_max=0
      cache.eviction_dirty_target=0
      cache.eviction_dirty_trigger=0
      cache.minimum=0
      checkpoint=on
      checkpoint.log_size=169
      checkpoint.wait=15
      chunk_cache=0
      chunk_cache.capacity=438
      chunk_cache.chunk_size=3
      chunk_cache.storage_path=off
      chunk_cache.type=off
      compact.free_space_target=16
      debug.checkpoint_retention=5
      debug.cursor_reposition=0
      debug.eviction=0
      debug.log_retention=2
      debug.realloc_exact=0
      debug.realloc_malloc=0
      debug.slow_checkpoint=0
      debug.table_logging=0
      debug.update_restore_evict=0
      disk.data_extend=0
      disk.direct_io=0
      disk.encryption=rotn-7
      disk.mmap=1
      disk.mmap_all=0
      file_manager.close_handle_minimum=27
      file_manager.close_idle_time=41
      file_manager.close_scan_interval=22
      format.abort=0
      format.independent_thread_rng=1
      format.major_timeout=0
      import=0
      logging=0
      logging.compression=none
      logging.file_max=207442
      logging.prealloc=1
      logging.remove=0
      ops.alter=0
      ops.compaction=0
      ops.hs_cursor=0
      ops.bound_cursor=1
      ops.prepare=0
      ops.random_cursor=0
      ops.salvage=0
      ops.throttle=0
      ops.throttle.sleep_us=355160
      ops.verify=0
      quiet=1
      random.data_seed=2487821
      random.extra_seed=2237942
      runs.in_memory=0
      runs.ops=0
      runs.predictable_replay=0
      runs.source=table
      runs.tables=12
      runs.threads=18
      runs.timer=10
      runs.verify_failure_dump=0
      statistics.mode=fast
      statistics_log.sources=off
      stress.aggressive_stash_free=0
      stress.aggressive_sweep=0
      stress.checkpoint=0
      stress.checkpoint_evict_page=0
      stress.checkpoint_prepare=0
      stress.compact_slow=0
      stress.evict_reposition=0
      stress.failpoint_eviction_split=0
      stress.failpoint_hs_delete_key_from_ts=1
      stress.hs_checkpoint_delay=0
      stress.hs_search=0
      stress.hs_sweep=0
      stress.prepare_resolution_1=0
      stress.sleep_before_read_overflow_onpage=0
      stress.split_1=0
      stress.split_2=0
      stress.split_3=0
      stress.split_4=0
      stress.split_5=0
      stress.split_6=0
      stress.split_7=0
      stress.split_8=0
      tiered_storage.flush_frequency=100
      tiered_storage.storage_source=dir_store
      transaction.implicit=0
      transaction.operation_timeout_ms=2000
      transaction.timestamps=1
      wiredtiger.config=off
      wiredtiger.rwlock=1
      wiredtiger.leak_memory=0
      ############################################
      #  TABLE PARAMETERS: table 1
      ############################################
      table1.btree.compression=lz4
      table1.btree.dictionary=0
      table1.btree.huffman_value=0
      table1.btree.internal_key_truncation=1
      table1.btree.internal_page_max=17
      table1.btree.key_max=32
      table1.btree.key_min=14
      table1.btree.leaf_page_max=9
      table1.btree.memory_page_max=2
      table1.btree.prefix_len=0
      table1.btree.prefix_compression=1
      table1.btree.prefix_compression_min=7
      table1.btree.reverse=0
      table1.btree.split_pct=65
      table1.btree.value_max=511
      table1.btree.value_min=13
      table1.disk.checksum=unencrypted
      table1.disk.firstfit=0
      table1.ops.pareto=0
      table1.ops.pareto.skew=64
      table1.ops.pct.delete=42
      table1.ops.pct.insert=2
      table1.ops.pct.modify=3
      table1.ops.pct.read=39
      table1.ops.pct.write=14
      table1.ops.truncate=1
      table1.runs.mirror=0
      table1.runs.rows=799288
      table1.runs.type=row-store
      ############################################
      #  TABLE PARAMETERS: table 2
      ############################################
      table2.btree.compression=lz4
      table2.btree.dictionary=0
      table2.btree.huffman_value=0
      table2.btree.internal_key_truncation=1
      table2.btree.internal_page_max=9
      table2.btree.leaf_page_max=14
      table2.btree.memory_page_max=1
      table2.btree.repeat_data_pct=22
      table2.btree.split_pct=53
      table2.btree.value_max=242
      table2.btree.value_min=16
      table2.disk.checksum=unencrypted
      table2.disk.firstfit=0
      table2.ops.pareto=1
      table2.ops.pareto.skew=35
      table2.ops.pct.delete=62
      table2.ops.pct.insert=13
      table2.ops.pct.modify=0
      table2.ops.pct.read=24
      table2.ops.pct.write=1
      table2.ops.truncate=1
      table2.runs.mirror=0
      table2.runs.rows=625110
      table2.runs.type=variable-length column-store
      ############################################
      #  TABLE PARAMETERS: table 3
      ############################################
      table3.btree.compression=zstd
      table3.btree.dictionary=1
      table3.btree.huffman_value=0
      table3.btree.internal_key_truncation=1
      table3.btree.internal_page_max=17
      table3.btree.key_max=107
      table3.btree.key_min=32
      table3.btree.leaf_page_max=9
      table3.btree.memory_page_max=2
      table3.btree.prefix_len=0
      table3.btree.prefix_compression=0
      table3.btree.prefix_compression_min=2
      table3.btree.reverse=0
      table3.btree.split_pct=50
      table3.btree.value_max=2512
      table3.btree.value_min=11
      table3.disk.checksum=unencrypted
      table3.disk.firstfit=0
      table3.ops.pareto=0
      table3.ops.pareto.skew=62
      table3.ops.pct.delete=0
      table3.ops.pct.insert=0
      table3.ops.pct.modify=0
      table3.ops.pct.read=0
      table3.ops.pct.write=100
      table3.ops.truncate=1
      table3.runs.mirror=0
      table3.runs.rows=711044
      table3.runs.type=row-store
      ############################################
      #  TABLE PARAMETERS: table 4
      ############################################
      table4.btree.compression=none
      table4.btree.dictionary=0
      table4.btree.huffman_value=0
      table4.btree.internal_key_truncation=1
      table4.btree.internal_page_max=10
      table4.btree.key_max=51
      table4.btree.key_min=26
      table4.btree.leaf_page_max=15
      table4.btree.memory_page_max=6
      table4.btree.prefix_len=0
      table4.btree.prefix_compression=1
      table4.btree.prefix_compression_min=4
      table4.btree.reverse=0
      table4.btree.split_pct=68
      table4.btree.value_max=2970
      table4.btree.value_min=19
      table4.disk.checksum=unencrypted
      table4.disk.firstfit=0
      table4.ops.pareto=0
      table4.ops.pareto.skew=13
      table4.ops.pct.delete=4
      table4.ops.pct.insert=12
      table4.ops.pct.modify=3
      table4.ops.pct.read=77
      table4.ops.pct.write=4
      table4.ops.truncate=1
      table4.runs.mirror=0
      table4.runs.rows=222482
      table4.runs.type=row-store
      ############################################
      #  TABLE PARAMETERS: table 5
      ############################################
      table5.btree.compression=zstd
      table5.btree.dictionary=0
      table5.btree.huffman_value=0
      table5.btree.internal_key_truncation=1
      table5.btree.internal_page_max=12
      table5.btree.leaf_page_max=10
      table5.btree.memory_page_max=1
      table5.btree.repeat_data_pct=61
      table5.btree.split_pct=66
      table5.btree.value_max=2591
      table5.btree.value_min=6
      table5.disk.checksum=unencrypted
      table5.disk.firstfit=0
      table5.ops.pareto=1
      table5.ops.pareto.skew=84
      table5.ops.pct.delete=20
      table5.ops.pct.insert=1
      table5.ops.pct.modify=1
      table5.ops.pct.read=64
      table5.ops.pct.write=14
      table5.ops.truncate=1
      table5.runs.mirror=0
      table5.runs.rows=78658
      table5.runs.type=variable-length column-store
      ############################################
      #  TABLE PARAMETERS: table 6
      ############################################
      table6.btree.compression=lz4
      table6.btree.dictionary=0
      table6.btree.huffman_value=0
      table6.btree.internal_key_truncation=1
      table6.btree.internal_page_max=11
      table6.btree.key_max=114
      table6.btree.key_min=31
      table6.btree.leaf_page_max=15
      table6.btree.memory_page_max=2
      table6.btree.prefix_len=0
      table6.btree.prefix_compression=1
      table6.btree.prefix_compression_min=5
      table6.btree.reverse=0
      table6.btree.split_pct=80
      table6.btree.value_max=751
      table6.btree.value_min=7
      table6.disk.checksum=on
      table6.disk.firstfit=0
      table6.ops.pareto=0
      table6.ops.pareto.skew=70
      table6.ops.pct.delete=57
      table6.ops.pct.insert=13
      table6.ops.pct.modify=2
      table6.ops.pct.read=9
      table6.ops.pct.write=19
      table6.ops.truncate=1
      table6.runs.mirror=0
      table6.runs.rows=589127
      table6.runs.type=row-store
      ############################################
      #  TABLE PARAMETERS: table 7
      ############################################
      table7.btree.bitcnt=2
      table7.btree.compression=zlib
      table7.btree.internal_key_truncation=1
      table7.btree.internal_page_max=13
      table7.btree.leaf_page_max=17
      table7.btree.memory_page_max=7
      table7.btree.split_pct=50
      table7.disk.checksum=on
      table7.disk.firstfit=0
      table7.ops.pareto=1
      table7.ops.pareto.skew=8
      table7.ops.pct.delete=8
      table7.ops.pct.insert=52
      table7.ops.pct.modify=29
      table7.ops.pct.read=0
      table7.ops.pct.write=11
      table7.ops.truncate=1
      table7.runs.mirror=0
      table7.runs.rows=321303
      table7.runs.type=fixed-length column-store
      ############################################
      #  TABLE PARAMETERS: table 8
      ############################################
      table8.btree.compression=lz4
      table8.btree.dictionary=1
      table8.btree.huffman_value=0
      table8.btree.internal_key_truncation=1
      table8.btree.internal_page_max=16
      table8.btree.leaf_page_max=16
      table8.btree.memory_page_max=6
      table8.btree.repeat_data_pct=81
      table8.btree.split_pct=79
      table8.btree.value_max=1951
      table8.btree.value_min=13
      table8.disk.checksum=unencrypted
      table8.disk.firstfit=0
      table8.ops.pareto=0
      table8.ops.pareto.skew=74
      table8.ops.pct.delete=15
      table8.ops.pct.insert=8
      table8.ops.pct.modify=3
      table8.ops.pct.read=2
      table8.ops.pct.write=72
      table8.ops.truncate=1
      table8.runs.mirror=0
      table8.runs.rows=60071
      table8.runs.type=variable-length column-store
      ############################################
      #  TABLE PARAMETERS: table 9
      ############################################
      table9.btree.compression=zlib
      table9.btree.dictionary=1
      table9.btree.huffman_value=0
      table9.btree.internal_key_truncation=1
      table9.btree.internal_page_max=9
      table9.btree.key_max=119
      table9.btree.key_min=31
      table9.btree.leaf_page_max=14
      table9.btree.memory_page_max=1
      table9.btree.prefix_len=0
      table9.btree.prefix_compression=1
      table9.btree.prefix_compression_min=1
      table9.btree.reverse=0
      table9.btree.split_pct=51
      table9.btree.value_max=668
      table9.btree.value_min=14
      table9.disk.checksum=unencrypted
      table9.disk.firstfit=0
      table9.ops.pareto=0
      table9.ops.pareto.skew=48
      table9.ops.pct.delete=20
      table9.ops.pct.insert=9
      table9.ops.pct.modify=1
      table9.ops.pct.read=70
      table9.ops.pct.write=0
      table9.ops.truncate=1
      table9.runs.mirror=0
      table9.runs.rows=656045
      table9.runs.type=row-store
      ############################################
      #  TABLE PARAMETERS: table 10
      ############################################
      table10.btree.compression=zlib
      table10.btree.dictionary=1
      table10.btree.huffman_value=0
      table10.btree.internal_key_truncation=1
      table10.btree.internal_page_max=10
      table10.btree.key_max=36
      table10.btree.key_min=23
      table10.btree.leaf_page_max=12
      table10.btree.memory_page_max=5
      table10.btree.prefix_len=0
      table10.btree.prefix_compression=1
      table10.btree.prefix_compression_min=6
      table10.btree.reverse=0
      table10.btree.split_pct=89
      table10.btree.value_max=2078
      table10.btree.value_min=18
      table10.disk.checksum=on
      table10.disk.firstfit=0
      table10.ops.pareto=0
      table10.ops.pareto.skew=75
      table10.ops.pct.delete=1
      table10.ops.pct.insert=0
      table10.ops.pct.modify=96
      table10.ops.pct.read=0
      table10.ops.pct.write=3
      table10.ops.truncate=1
      table10.runs.mirror=0
      table10.runs.rows=680388
      table10.runs.type=row-store
      ############################################
      #  TABLE PARAMETERS: table 11
      ############################################
      table11.btree.compression=zlib
      table11.btree.dictionary=1
      table11.btree.huffman_value=1
      table11.btree.internal_key_truncation=1
      table11.btree.internal_page_max=11
      table11.btree.key_max=126
      table11.btree.key_min=25
      table11.btree.leaf_page_max=13
      table11.btree.memory_page_max=4
      table11.btree.prefix_len=0
      table11.btree.prefix_compression=1
      table11.btree.prefix_compression_min=3
      table11.btree.reverse=0
      table11.btree.split_pct=60
      table11.btree.value_max=1305
      table11.btree.value_min=8
      table11.disk.checksum=on
      table11.disk.firstfit=0
      table11.ops.pareto=0
      table11.ops.pareto.skew=62
      table11.ops.pct.delete=1
      table11.ops.pct.insert=21
      table11.ops.pct.modify=4
      table11.ops.pct.read=64
      table11.ops.pct.write=10
      table11.ops.truncate=1
      table11.runs.mirror=0
      table11.runs.rows=986183
      table11.runs.type=row-store
      ############################################
      #  TABLE PARAMETERS: table 12
      ############################################
      table12.btree.compression=zstd
      table12.btree.dictionary=0
      table12.btree.huffman_value=1
      table12.btree.internal_key_truncation=1
      table12.btree.internal_page_max=17
      table12.btree.key_max=52
      table12.btree.key_min=32
      table12.btree.leaf_page_max=14
      table12.btree.memory_page_max=4
      table12.btree.prefix_len=0
      table12.btree.prefix_compression=0
      table12.btree.prefix_compression_min=4
      table12.btree.reverse=0
      table12.btree.split_pct=66
      table12.btree.value_max=3440
      table12.btree.value_min=13
      table12.disk.checksum=on
      table12.disk.firstfit=1
      table12.ops.pareto=0
      table12.ops.pareto.skew=15
      table12.ops.pct.delete=0
      table12.ops.pct.insert=8
      table12.ops.pct.modify=16
      table12.ops.pct.read=4
      table12.ops.pct.write=72
      table12.ops.truncate=1
      table12.runs.mirror=0
      table12.runs.rows=723020
      table12.runs.type=row-store 

            Assignee:
            backlog-server-storage-engines [DO NOT USE] Backlog - Storage Engines Team
            Reporter:
            siddhartha.mahajan@mongodb.com Sid Mahajan
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

              Created:
              Updated: