|
1. systemLog:
destination: file
path: "/storage/logs/mongodb.log"
logAppend: true
storage:
dbPath: "/storage/data"
engine: "wiredTiger"
journal:
enabled: false
indexBuildRetry: false
net:
port: 27017
2. I can share, program is R script. But I think it will be hard to understand underlying logic. It used *rmongodb* (https://github.com/mongosoup/rmongodb) package (I'm maintainer of this package), which is based on *mongo-c-driver 0.8.0* (https://github.com/mongodb/mongo-c-driver-legacy). Average size of document is 5124 bytes. Also I noticed one more strange thing. I have server mongod machine with 16 cores. If I run my script in 32 threads (from another client machine), updates are slow ~ 600-700 sec per 32mb chunk. db cpu fully utilized and disk almost all not utilizes. If I reduce threads to 16, one chunk update takes ~170 sec and disk utilized at ~30%.
Here is stats of my collection:
/* 0 */
{
"ns" : "mydb.normalized_profiles",
"count" : 214327259,
"size" : 1098340528552,
"avgObjSize" : 5124,
"storageSize" : 573728669696,
"capped" : false,
"wiredTiger" : {
"metadata" : {},
"creationString" : "allocation_size=4KB,app_metadata=(),block_allocation=best,block_compressor=snappy,cache_resident=0,checkpoint=(WiredTigerCheckpoint.20532=(addr=\"01e408289e6681e424dc5c47e407afe8c6cf93e4435dea58e407afca0bcf0be410adb561808080e58594e49fc0e56b93718fc0\",order=20532,time=1420051642,size=462035202048,write_gen=573841289)),checkpoint_lsn=(48018,59367168),checksum=uncompressed,collator=,columns=,dictionary=0,format=btree,huffman_key=,huffman_value=,id=6,internal_item_max=0,internal_key_truncate=,internal_page_max=4KB,key_format=q,key_gap=10,leaf_item_max=0,leaf_page_max=32KB,memory_page_max=100m,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=0,prefix_compression_min=4,split_pct=75,value_format=u,version=(major=1,minor=1)",
"type" : "file",
"uri" : "statistics:table:collection-0-3607676398058767653",
"LSM" :
{
"bloom filters in the LSM tree" : 0,
"bloom filter false positives" : 0,
"bloom filter hits" : 0,
"bloom filter misses" : 0,
"bloom filter pages evicted from cache" : 0,
"bloom filter pages read into cache" : 0,
"total size of bloom filters" : 0,
"sleep for LSM checkpoint throttle" : 0,
"chunks in the LSM tree" : 0,
"highest merge generation in the LSM tree" : 0,
"queries that could have benefited from a Bloom filter that did not exist" : 0,
"sleep for LSM merge throttle" : 0
}
,
"block-manager" :
{
"file allocation unit size" : 4096,
"blocks allocated" : 69485067,
"checkpoint size" : 462035202048,
"allocations requiring file extension" : 0,
"blocks freed" : 69346343,
"file magic number" : 120897,
"file major version number" : 1,
"minor version number" : 0,
"file bytes available for reuse" : 111722827776,
"file size in bytes" : 573728669696
}
,
"btree" :
{
"column-store variable-size deleted values" : 0,
"column-store fixed-size leaf pages" : 0,
"column-store internal pages" : 0,
"column-store variable-size leaf pages" : 0,
"pages rewritten by compaction" : 0,
"number of key/value pairs" : 0,
"fixed-record size" : 0,
"maximum tree depth" : 9766,
"maximum internal page key size" : 307,
"maximum internal page size" : 4096,
"maximum leaf page key size" : 2457,
"maximum leaf page size" : 32768,
"maximum leaf page value size" : 12288,
"overflow pages" : 0,
"row-store internal pages" : 0,
"row-store leaf pages" : 0
}
,
"cache" :
{
"bytes read into cache" : NumberLong(1540641708376),
"bytes written from cache" : 965856098800,
"checkpoint blocked page eviction" : 975677,
"unmodified pages evicted" : 11021536,
"modified pages evicted" : 31604075,
"data source pages selected for eviction unable to be evicted" : 1114534,
"hazard pointer blocked page eviction" : 16036,
"internal pages evicted" : 35,
"in-memory page splits" : 0,
"overflow values cached in memory" : 393138,
"pages read into cache" : 69265263,
"overflow pages read into cache" : 25016375,
"pages written from cache" : 69484368
}
,
"compression" :
{
"raw compression call failed, no additional data available" : 0,
"raw compression call failed, additional data available" : 0,
"raw compression call succeeded" : 0,
"compressed pages read" : 68346447,
"compressed pages written" : 43253057,
"page written failed to compress" : 46,
"page written was too small to compress" : 26230866
}
,
"cursor" :
{
"create calls" : 295,
"insert calls" : 0,
"bulk-loaded cursor-insert calls" : 0,
"cursor-insert key and value bytes inserted" : 0,
"next calls" : 0,
"prev calls" : 1,
"remove calls" : 0,
"cursor-remove key bytes removed" : 0,
"reset calls" : 142065177,
"search calls" : 142031422,
"search near calls" : 0,
"update calls" : 35235635,
"cursor-update value bytes updated" : 258994311365
}
,
"reconciliation" :
{
"dictionary matches" : 0,
"internal page multi-block writes" : 2885,
"leaf page multi-block writes" : 33605,
"maximum blocks required for a page" : 3,
"internal-page overflow keys" : 0,
"leaf-page overflow keys" : 0,
"overflow values written" : 7116814,
"pages deleted" : 0,
"page checksum matches" : 14064,
"page reconciliation calls" : 63221524,
"page reconciliation calls for eviction" : 7038343,
"leaf page key bytes discarded using prefix compression" : 0,
"internal page key bytes discarded using suffix compression" : 3903403
}
,
"session" :
{
"object compaction" : 0,
"open cursor count" : 309
}
,
"transaction" :
{
"update conflicts" : 0
}
},
"nindexes" : 3,
"indexDetails" : {
"id" : {
"metadata" : {
"formatVersion" : 1,
"infoObj" : "{ \"v\" : 1, \"key\" :
{ \"_id\" : 1 }
, \"name\" : \"id\", \"ns\" : \"mydb.normalized_profiles\" }"
},
"creationString" : "allocation_size=4KB,app_metadata=(formatVersion=1,infoObj={ \"v\" : 1, \"key\" :
{ \"_id\" : 1 }
, \"name\" : \"id\", \"ns\" : \"mydb.normalized_profiles\" }),block_allocation=best,block_compressor=,cache_resident=0,checkpoint=(WiredTigerCheckpoint.2=(addr=\"01e30a80b981e453055a68e30a80ba81e45b113e14808080808080e4aa0f9fc0e4aa0f7fc0\",order=2,time=1419582444,size=2853150720,write_gen=233845)),checkpoint_lsn=(33753,92672),checksum=uncompressed,collator=mongo_index,columns=,dictionary=0,format=btree,huffman_key=,huffman_value=,id=77,internal_item_max=0,internal_key_max=0,internal_key_truncate=,internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=16k,leaf_value_max=0,memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=,prefix_compression_min=4,split_pct=75,value_format=u,version=(major=1,minor=1)",
"type" : "file",
"uri" : "statistics:table:index-1-1389960491970549037",
"LSM" :
{
"bloom filters in the LSM tree" : 0,
"bloom filter false positives" : 0,
"bloom filter hits" : 0,
"bloom filter misses" : 0,
"bloom filter pages evicted from cache" : 0,
"bloom filter pages read into cache" : 0,
"total size of bloom filters" : 0,
"sleep for LSM checkpoint throttle" : 0,
"chunks in the LSM tree" : 0,
"highest merge generation in the LSM tree" : 0,
"queries that could have benefited from a Bloom filter that did not exist" : 0,
"sleep for LSM merge throttle" : 0
}
,
"block-manager" :
{
"file allocation unit size" : 4096,
"blocks allocated" : 0,
"checkpoint size" : 2853150720,
"allocations requiring file extension" : 0,
"blocks freed" : 0,
"file magic number" : 120897,
"file major version number" : 1,
"minor version number" : 0,
"file bytes available for reuse" : 0,
"file size in bytes" : 2853158912
}
,
"btree" :
{
"column-store variable-size deleted values" : 0,
"column-store fixed-size leaf pages" : 0,
"column-store internal pages" : 0,
"column-store variable-size leaf pages" : 0,
"pages rewritten by compaction" : 0,
"number of key/value pairs" : 0,
"fixed-record size" : 0,
"maximum tree depth" : 5,
"maximum internal page key size" : 307,
"maximum internal page size" : 4096,
"maximum leaf page key size" : 1228,
"maximum leaf page size" : 16384,
"maximum leaf page value size" : 6144,
"overflow pages" : 0,
"row-store internal pages" : 0,
"row-store leaf pages" : 0
}
,
"cache" :
{
"bytes read into cache" : 395709495706,
"bytes written from cache" : 0,
"checkpoint blocked page eviction" : 0,
"unmodified pages evicted" : 32225190,
"modified pages evicted" : 0,
"data source pages selected for eviction unable to be evicted" : 697,
"hazard pointer blocked page eviction" : 697,
"internal pages evicted" : 0,
"in-memory page splits" : 0,
"overflow values cached in memory" : 0,
"pages read into cache" : 32220500,
"overflow pages read into cache" : 0,
"pages written from cache" : 0
}
,
"compression" :
{
"raw compression call failed, no additional data available" : 0,
"raw compression call failed, additional data available" : 0,
"raw compression call succeeded" : 0,
"compressed pages read" : 0,
"compressed pages written" : 0,
"page written failed to compress" : 0,
"page written was too small to compress" : 0
}
,
"cursor" :
{
"create calls" : 295,
"insert calls" : 0,
"bulk-loaded cursor-insert calls" : 0,
"cursor-insert key and value bytes inserted" : 0,
"next calls" : 0,
"prev calls" : 0,
"remove calls" : 0,
"cursor-remove key bytes removed" : 0,
"reset calls" : 35228438,
"search calls" : 0,
"search near calls" : 35235316,
"update calls" : 0,
"cursor-update value bytes updated" : 0
}
,
"reconciliation" :
{
"dictionary matches" : 0,
"internal page multi-block writes" : 0,
"leaf page multi-block writes" : 0,
"maximum blocks required for a page" : 0,
"internal-page overflow keys" : 0,
"leaf-page overflow keys" : 0,
"overflow values written" : 0,
"pages deleted" : 0,
"page checksum matches" : 0,
"page reconciliation calls" : 0,
"page reconciliation calls for eviction" : 0,
"leaf page key bytes discarded using prefix compression" : 0,
"internal page key bytes discarded using suffix compression" : 0
}
,
"session" :
{
"object compaction" : 0,
"open cursor count" : 307
}
,
"transaction" :
{
"update conflicts" : 0
}
},
"source_1_sourceId_1" : {
"metadata" : {
"formatVersion" : 1,
"infoObj" : "{ \"v\" : 1, \"unique\" : true, \"key\" :
{ \"source\" : 1, \"sourceId\" : 1 }
, \"name\" : \"source_1_sourceId_1\", \"ns\" : \"mydb.normalized_profiles\", \"background\" : true }"
},
"creationString" : "allocation_size=4KB,app_metadata=(formatVersion=1,infoObj={ \"v\" : 1, \"unique\" : true, \"key\" :
{ \"source\" : 1, \"sourceId\" : 1 }
, \"name\" : \"source_1_sourceId_1\", \"ns\" : \"mydb.normalized_profiles\", \"background\" : true }),block_allocation=best,block_compressor=,cache_resident=0,checkpoint=(WiredTigerCheckpoint.3=(addr=\"01e3104c4281e451bf4d34e3104c4381e4cab0e7cce3104c4481e4de2d9e7d808080e50106c83fc0e50106c76fc0\",order=3,time=1419672432,size=4408709120,write_gen=362485)),checkpoint_lsn=(33800,547072),checksum=uncompressed,collator=mongo_index,columns=,dictionary=0,format=btree,huffman_key=,huffman_value=,id=78,internal_item_max=0,internal_key_max=0,internal_key_truncate=,internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=16k,leaf_value_max=0,memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=,prefix_compression_min=4,split_pct=75,value_format=u,version=(major=1,minor=1)",
"type" : "file",
"uri" : "statistics:table:index-2-1389960491970549037",
"LSM" :
{
"bloom filters in the LSM tree" : 0,
"bloom filter false positives" : 0,
"bloom filter hits" : 0,
"bloom filter misses" : 0,
"bloom filter pages evicted from cache" : 0,
"bloom filter pages read into cache" : 0,
"total size of bloom filters" : 0,
"sleep for LSM checkpoint throttle" : 0,
"chunks in the LSM tree" : 0,
"highest merge generation in the LSM tree" : 0,
"queries that could have benefited from a Bloom filter that did not exist" : 0,
"sleep for LSM merge throttle" : 0
}
,
"block-manager" :
{
"file allocation unit size" : 4096,
"blocks allocated" : 0,
"checkpoint size" : 4408709120,
"allocations requiring file extension" : 0,
"blocks freed" : 0,
"file magic number" : 120897,
"file major version number" : 1,
"minor version number" : 0,
"file bytes available for reuse" : 45056,
"file size in bytes" : 4408762368
}
,
"btree" :
{
"column-store variable-size deleted values" : 0,
"column-store fixed-size leaf pages" : 0,
"column-store internal pages" : 0,
"column-store variable-size leaf pages" : 0,
"pages rewritten by compaction" : 0,
"number of key/value pairs" : 0,
"fixed-record size" : 0,
"maximum tree depth" : 5,
"maximum internal page key size" : 307,
"maximum internal page size" : 4096,
"maximum leaf page key size" : 1228,
"maximum leaf page size" : 16384,
"maximum leaf page value size" : 6144,
"overflow pages" : 0,
"row-store internal pages" : 0,
"row-store leaf pages" : 0
}
,
"cache" :
{
"bytes read into cache" : 5161701415,
"bytes written from cache" : 0,
"checkpoint blocked page eviction" : 0,
"unmodified pages evicted" : 424347,
"modified pages evicted" : 0,
"data source pages selected for eviction unable to be evicted" : 890474,
"hazard pointer blocked page eviction" : 890433,
"internal pages evicted" : 5304,
"in-memory page splits" : 0,
"overflow values cached in memory" : 0,
"pages read into cache" : 424805,
"overflow pages read into cache" : 0,
"pages written from cache" : 0
}
,
"compression" :
{
"raw compression call failed, no additional data available" : 0,
"raw compression call failed, additional data available" : 0,
"raw compression call succeeded" : 0,
"compressed pages read" : 0,
"compressed pages written" : 0,
"page written failed to compress" : 0,
"page written was too small to compress" : 0
}
,
"cursor" :
{
"create calls" : 271,
"insert calls" : 0,
"bulk-loaded cursor-insert calls" : 0,
"cursor-insert key and value bytes inserted" : 0,
"next calls" : 286977364,
"prev calls" : 0,
"remove calls" : 0,
"cursor-remove key bytes removed" : 0,
"reset calls" : 22127,
"search calls" : 0,
"search near calls" : 22130,
"update calls" : 0,
"cursor-update value bytes updated" : 0
}
,
"reconciliation" :
{
"dictionary matches" : 0,
"internal page multi-block writes" : 0,
"leaf page multi-block writes" : 0,
"maximum blocks required for a page" : 0,
"internal-page overflow keys" : 0,
"leaf-page overflow keys" : 0,
"overflow values written" : 0,
"pages deleted" : 0,
"page checksum matches" : 0,
"page reconciliation calls" : 0,
"page reconciliation calls for eviction" : 0,
"leaf page key bytes discarded using prefix compression" : 0,
"internal page key bytes discarded using suffix compression" : 0
}
,
"session" :
{
"object compaction" : 0,
"open cursor count" : 283
}
,
"transaction" :
{
"update conflicts" : 0
}
},
"email.data.value_1" : {
"metadata" : {
"formatVersion" : 1,
"infoObj" : "{ \"v\" : 1, \"key\" :
{ \"email.data.value\" : 1 }
, \"name\" : \"email.data.value_1\", \"ns\" : \"mydb.normalized_profiles\" }"
},
"creationString" : "allocation_size=4KB,app_metadata=(formatVersion=1,infoObj={ \"v\" : 1, \"key\" :
{ \"email.data.value\" : 1 }
, \"name\" : \"email.data.value_1\", \"ns\" : \"mydb.normalized_profiles\" }),block_allocation=best,block_compressor=,cache_resident=0,checkpoint=(WiredTigerCheckpoint.2=(addr=\"01e30e7d4281e42d5b3f1ce30e7d4381e4134ea548808080808080e4e9d82fc0e4e9d80fc0\",order=2,time=1419583082,size=3923259392,write_gen=322267)),checkpoint_lsn=(33753,98176),checksum=uncompressed,collator=mongo_index,columns=,dictionary=0,format=btree,huffman_key=,huffman_value=,id=79,internal_item_max=0,internal_key_max=0,internal_key_truncate=,internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=16k,leaf_value_max=0,memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=,prefix_compression_min=4,split_pct=75,value_format=u,version=(major=1,minor=1)",
"type" : "file",
"uri" : "statistics:table:index-3-1389960491970549037",
"LSM" :
{
"bloom filters in the LSM tree" : 0,
"bloom filter false positives" : 0,
"bloom filter hits" : 0,
"bloom filter misses" : 0,
"bloom filter pages evicted from cache" : 0,
"bloom filter pages read into cache" : 0,
"total size of bloom filters" : 0,
"sleep for LSM checkpoint throttle" : 0,
"chunks in the LSM tree" : 0,
"highest merge generation in the LSM tree" : 0,
"queries that could have benefited from a Bloom filter that did not exist" : 0,
"sleep for LSM merge throttle" : 0
}
,
"block-manager" :
{
"file allocation unit size" : 4096,
"blocks allocated" : 0,
"checkpoint size" : 3923259392,
"allocations requiring file extension" : 0,
"blocks freed" : 0,
"file magic number" : 120897,
"file major version number" : 1,
"minor version number" : 0,
"file bytes available for reuse" : 0,
"file size in bytes" : 3923267584
}
,
"btree" :
{
"column-store variable-size deleted values" : 0,
"column-store fixed-size leaf pages" : 0,
"column-store internal pages" : 0,
"column-store variable-size leaf pages" : 0,
"pages rewritten by compaction" : 0,
"number of key/value pairs" : 0,
"fixed-record size" : 0,
"maximum tree depth" : 0,
"maximum internal page key size" : 307,
"maximum internal page size" : 4096,
"maximum leaf page key size" : 1228,
"maximum leaf page size" : 16384,
"maximum leaf page value size" : 6144,
"overflow pages" : 0,
"row-store internal pages" : 0,
"row-store leaf pages" : 0
}
,
"cache" :
{
"bytes read into cache" : 3263,
"bytes written from cache" : 0,
"checkpoint blocked page eviction" : 0,
"unmodified pages evicted" : 0,
"modified pages evicted" : 0,
"data source pages selected for eviction unable to be evicted" : 0,
"hazard pointer blocked page eviction" : 0,
"internal pages evicted" : 0,
"in-memory page splits" : 0,
"overflow values cached in memory" : 0,
"pages read into cache" : 1,
"overflow pages read into cache" : 0,
"pages written from cache" : 0
}
,
"compression" :
{
"raw compression call failed, no additional data available" : 0,
"raw compression call failed, additional data available" : 0,
"raw compression call succeeded" : 0,
"compressed pages read" : 0,
"compressed pages written" : 0,
"page written failed to compress" : 0,
"page written was too small to compress" : 0
}
,
"cursor" :
{
"create calls" : 0,
"insert calls" : 0,
"bulk-loaded cursor-insert calls" : 0,
"cursor-insert key and value bytes inserted" : 0,
"next calls" : 0,
"prev calls" : 0,
"remove calls" : 0,
"cursor-remove key bytes removed" : 0,
"reset calls" : 0,
"search calls" : 0,
"search near calls" : 0,
"update calls" : 0,
"cursor-update value bytes updated" : 0
}
,
"reconciliation" :
{
"dictionary matches" : 0,
"internal page multi-block writes" : 0,
"leaf page multi-block writes" : 0,
"maximum blocks required for a page" : 0,
"internal-page overflow keys" : 0,
"leaf-page overflow keys" : 0,
"overflow values written" : 0,
"pages deleted" : 0,
"page checksum matches" : 0,
"page reconciliation calls" : 0,
"page reconciliation calls for eviction" : 0,
"leaf page key bytes discarded using prefix compression" : 0,
"internal page key bytes discarded using suffix compression" : 0
}
,
"session" :
{
"object compaction" : 0,
"open cursor count" : 0
}
,
"transaction" :
{
"update conflicts" : 0
}
}
},
"totalIndexSize" : 11185188864,
"indexSizes" :
{
"_id_" : 2853158912,
"source_1_sourceId_1" : 4408762368,
"email.data.value_1" : 3923267584
}
,
"ok" : 1
}
|