LSM tests failing with stalled eviction.

[1508957349:759453][37866:0x3fffaa39f1b0], t, eviction-server: Cache stuck for too long, giving up: Connection timed out

t: process 37866

0x10037e1b750:=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

0x10037e1b750:transaction state dump

0x10037e1b750:current ID: 153672

0x10037e1b750:last running ID: 153672

0x10037e1b750:oldest ID: 153570

0x10037e1b750:commit timestamp: 1891

0x10037e1b750:oldest timestamp: 1825

0x10037e1b750:pinned timestamp: 1825

0x10037e1b750:stable timestamp: 0

0x10037e1b750:has_commit_timestamp: yes

0x10037e1b750:has_oldest_timestamp: yes

0x10037e1b750:has_pinned_timestamp: yes

0x10037e1b750:has_stable_timestamp: no

0x10037e1b750:oldest_is_pinned: yes

0x10037e1b750:stable_is_pinned: no

0x10037e1b750:checkpoint running: no

0x10037e1b750:checkpoint generation: 14

0x10037e1b750:checkpoint pinned ID: 0

0x10037e1b750:checkpoint txn ID: 0

0x10037e1b750:oldest named snapshot ID: 0

0x10037e1b750:session count: 43

0x10037e1b750:Transaction state of active sessions:

0x10037e1b750:ID: 0, pinned ID: 153570, metadata pinned ID: 149774, name: WT_CURSOR.search_near

0x10037e21510:mod count: 0, snap min: 149774, snap max: 153653, commit_timestamp: 1825, first_commit_timestamp: 1825, read_timestamp: 0, flags: 0x00000008, isolation: WT_ISO_SNAPSHOT

0x10037e1b750:=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

0x10037e1b750:cache dump

0x10037e1b750:cache full: yes

0x10037e1b750:cache clean check: yes (116%)

0x10037e1b750:cache dirty check: no (0%)

0x10037e1b750:file:wt-000031.lsm(checkpoint=WiredTigerCheckpoint.2):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000044.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 0/1 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000043.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000042.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 2MB, 1/0 clean/dirty pages, 2/0 clean/dirty MB, 2MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000028.bf(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000031.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000041.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 3MB, 1/0 clean/dirty pages, 3/0 clean/dirty MB, 3MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000026.bf(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000040.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 2MB, 1/0 clean/dirty pages, 2/0 clean/dirty MB, 2MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000025.bf(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000037.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000037.bf(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000039.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 3MB, 1/0 clean/dirty pages, 3/0 clean/dirty MB, 3MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000038.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 3MB, 1/0 clean/dirty pages, 3/0 clean/dirty MB, 3MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000036.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 3MB, 1/0 clean/dirty pages, 3/0 clean/dirty MB, 3MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000035.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 3MB, 1/0 clean/dirty pages, 3/0 clean/dirty MB, 3MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000034.lsm(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 3MB, 1/0 clean/dirty pages, 3/0 clean/dirty MB, 3MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000033.lsm(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 3MB, 1/0 clean/dirty pages, 3/0 clean/dirty MB, 3MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000032.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000032.bf(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000030.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000029.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000028.lsm(checkpoint=WiredTigerCheckpoint.2):

0x10037e1b750:internal: 3 pages, 0MB, 3/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000026.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 3 pages, 0MB, 3/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000025.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 3 pages, 0MB, 3/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000027.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000027.bf(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000021.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000021.bf(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000015.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 3 pages, 0MB, 3/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000015.bf(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:wt-000008.lsm(checkpoint=WiredTigerCheckpoint.1):

0x10037e1b750:internal: 2 pages, 0MB, 2/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:WiredTigerLAS.wt(<live>)eviction disabled at open:

0x10037e1b750:internal: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 1 pages, 0MB, 1/0 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:file:WiredTiger.wt(<live>):

0x10037e1b750:internal: 1 pages, 0MB, 0/1 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:leaf: 3 pages, 0MB, 0/3 clean/dirty pages, 0/0 clean/dirty MB, 0MB max page, 0MB max dirty page

0x10037e1b750:cache dump: total found: 34MB vs tracked inuse 34MB

0x10037e1b750:total dirty bytes: 0MB

[1508957349:772967][37866:0x3fffaa39f1b0], t, eviction-server: cache eviction thread error: Connection timed out

[1508957349:772980][37866:0x3fffaa39f1b0], t, eviction-server: the process must exit and restart: WT_PANIC: WiredTiger library panic

[1508957349:772985][37866:0x3fffaa39f1b0], t, eviction-server: aborting WiredTiger library

############################################

#  RUN PARAMETERS

############################################

abort=0

alter=0

auto_throttle=1

backups=0

bitcnt=5

bloom=1

bloom_bit_count=35

bloom_hash_count=20

bloom_oldest=0

cache=30

checkpoints=on

checkpoint_log_size=93

checkpoint_wait=54

checksum=uncompressed

chunk_size=1

compaction=0

compression=zlib

data_extend=1

data_source=lsm

delete_pct=79

dictionary=0

direct_io=0

encryption=none

evict_max=3

file_type=row-store

firstfit=0

huffman_key=0

huffman_value=0

independent_thread_rng=1

in_memory=0

insert_pct=6

internal_key_truncation=1

internal_page_max=9

isolation=random

key_gap=8

key_max=256

key_min=256

leaf_page_max=9

leak_memory=0

logging=1

logging_archive=1

logging_compression=zlib

logging_file_max=156885

logging_prealloc=1

long_running_txn=0

lsm_worker_threads=3

merge_max=7

mmap=1

modify_pct=2

ops=100000

prefix_compression=1

prefix_compression_min=7

quiet=1

read_pct=1

rebalance=1

repeat_data_pct=76

reverse=1

rows=100000

runs=1

salvage=1

split_pct=51

statistics=0

statistics_server=0

threads=11

timer=360

transaction_timestamps=1

transaction-frequency=87

value_max=2082

value_min=10

verify=1

wiredtiger_config=

write_pct=12

############################################