labels = array([], dtype=float64) src/timeaxis.c:419: TimeAxis(index=0, labels=array([], dtype=float64), sortedness=None, uniqueness=None) src/axis.c:418: axis_setlabels_internal: LabelAxis(index=0, labels=array([], dtype=object)) src/axis.c:485: LabelAxis(index=1, labels=array([], dtype=object)) src/sandwich.c:187: sa_init: axindex = 1 src/sandwich.c:237: Calling __sandwich_finalize__ frame_finalize 2015-12-18 14:13:05,391 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,392 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,392 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,393 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,394 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,395 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,397 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,399 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,402 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,402 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,404 INFO [arctic.arctic] [Thread-3 46007-47907104479552] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,406 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,411 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,414 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,414 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,417 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,418 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,418 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,421 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,421 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,421 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,423 DEBUG [root] [Thread-3 46007-47907104479552] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,432 DEBUG [root] [Thread-3 46007-47907104479552] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,426 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,430 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,430 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,429 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,433 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,435 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,438 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,439 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,440 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,443 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,445 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,444 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,448 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,448 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,448 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,449 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,452 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,452 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,454 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,457 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,457 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,460 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,462 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,465 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,466 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,466 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,467 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,468 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,469 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,470 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,471 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,473 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,477 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,478 INFO [arctic.arctic] [Thread-15 46007-47907508160832] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,478 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,479 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,484 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,484 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,486 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <105> - starting 2015-12-18 14:13:05,487 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,487 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,488 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <66> - Stopping for: 3.15877560008 2015-12-18 14:13:05,490 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,491 DEBUG [root] [Thread-15 46007-47907508160832] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,492 INFO [arctic.arctic] [Thread-19 46007-47907637193024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,494 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,495 INFO [arctic.arctic] [Thread-16 46007-47907561662784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,496 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,502 DEBUG [root] [Thread-15 46007-47907508160832] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,503 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:05,510 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,512 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,514 DEBUG [root] [Thread-19 46007-47907637193024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,514 DEBUG [root] [Thread-16 46007-47907561662784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,516 DEBUG [root] [Thread-19 46007-47907637193024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,517 DEBUG [root] [Thread-16 46007-47907561662784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,518 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,519 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:05,519 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:05,521 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:08,018 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(14) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/14 -> socket:[26011094] 2015-12-18 14:13:08,530 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(4) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/4 -> socket:[26011194] 2015-12-18 14:13:08,655 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <69> - Running for: 1.05993379584 2015-12-18 14:13:08,680 ERROR [root] [Thread-13 46007-47907444169024] events._error <135> - Failed to send event ahl_mongo: [Errno 32] Broken pipe Traceback (most recent call last): File "/users/is/jblackburn/pyenvs/research/lib/python2.7/site-packages/ahl.diags_core-1.65.0.dev1-py2.7.egg/ahl/diagscore/events.py", line 166, in _log _socket().send(data) error: [Errno 32] Broken pipe 2015-12-18 14:13:08,681 ERROR [root] [Thread-6 46007-47907135961408] events._error <135> - Failed to send event ahl_mongo: [Errno 32] Broken pipe Traceback (most recent call last): File "/users/is/jblackburn/pyenvs/research/lib/python2.7/site-packages/ahl.diags_core-1.65.0.dev1-py2.7.egg/ahl/diagscore/events.py", line 166, in _log _socket().send(data) error: [Errno 32] Broken pipe 2015-12-18 14:13:08,682 ERROR [root] [Thread-5 46007-47907125467456] events._error <135> - Failed to send event ahl_mongo: [Errno 32] Broken pipe Traceback (most recent call last): File "/users/is/jblackburn/pyenvs/research/lib/python2.7/site-packages/ahl.diags_core-1.65.0.dev1-py2.7.egg/ahl/diagscore/events.py", line 166, in _log _socket().send(data) error: [Errno 32] Broken pipe 2015-12-18 14:13:08,682 ERROR [root] [Thread-13 46007-47907444169024] events._error <137> - Too many errors, error logging disabled for Event logging. Traceback (most recent call last): File "/users/is/jblackburn/pyenvs/research/lib/python2.7/site-packages/ahl.diags_core-1.65.0.dev1-py2.7.egg/ahl/diagscore/events.py", line 166, in _log _socket().send(data) error: [Errno 32] Broken pipe 2015-12-18 14:13:08,683 ERROR [root] [Thread-6 46007-47907135961408] events._error <137> - Too many errors, error logging disabled for Event logging. Traceback (most recent call last): File "/users/is/jblackburn/pyenvs/research/lib/python2.7/site-packages/ahl.diags_core-1.65.0.dev1-py2.7.egg/ahl/diagscore/events.py", line 166, in _log _socket().send(data) error: [Errno 32] Broken pipe 2015-12-18 14:13:08,684 ERROR [root] [Thread-5 46007-47907125467456] events._error <137> - Too many errors, error logging disabled for Event logging. Traceback (most recent call last): File "/users/is/jblackburn/pyenvs/research/lib/python2.7/site-packages/ahl.diags_core-1.65.0.dev1-py2.7.egg/ahl/diagscore/events.py", line 166, in _log _socket().send(data) error: [Errno 32] Broken pipe 2015-12-18 14:13:08,689 DEBUG [arctic.decorators] [Thread-6 46007-47907135961408] decorators._handle_error <88> - _conn [localhost:27888: [Errno 32] Broken pipe], retrying 1 2015-12-18 14:13:08,702 DEBUG [root] [Thread-15 46007-47907508160832] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,702 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,705 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,706 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,710 DEBUG [root] [Thread-15 46007-47907508160832] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,713 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,717 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,719 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,720 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,724 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,734 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,736 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,738 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,743 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,745 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,745 DEBUG [root] [Thread-19 46007-47907637193024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,749 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,751 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,751 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,751 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,752 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,755 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,757 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,759 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,763 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,763 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,766 DEBUG [root] [Thread-19 46007-47907637193024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,772 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,772 DEBUG [root] [Thread-3 46007-47907104479552] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,779 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,785 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,788 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,795 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,797 DEBUG [root] [Thread-16 46007-47907561662784] auth.authenticate <42> - Auth Error localhost:27888: [Errno 32] Broken pipe 2015-12-18 14:13:08,845 DEBUG [root] [Thread-3 46007-47907104479552] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,846 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,847 DEBUG [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - _conn [localhost:27888: [Errno 32] Broken pipe], retrying 1 2015-12-18 14:13:08,856 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:08,967 DEBUG [root] [Thread-16 46007-47907561662784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:08,971 DEBUG [root] [Thread-16 46007-47907561662784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,001 DEBUG [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - get_library_metadata [localhost:27888: [Errno 32] Broken pipe], retrying 1 2015-12-18 14:13:09,145 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,161 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,189 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,190 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,200 INFO [arctic.arctic] [Thread-15 46007-47907508160832] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,215 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,221 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,222 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,223 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(18) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/18 -> socket:[26011208] 2015-12-18 14:13:09,247 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,252 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,253 DEBUG [arctic.store.version_store] [Thread-15 46007-47907508160832] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,256 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,259 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,271 DEBUG [arctic.store.version_store] [Thread-17 46007-47907583691072] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,275 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,279 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,286 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,290 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,296 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,298 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,300 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,304 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,326 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,331 INFO [arctic.arctic] [Thread-19 46007-47907637193024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,331 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,330 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,341 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,342 DEBUG [arctic.store.version_store] [Thread-11 46007-47907348707648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,348 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,352 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,359 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,363 DEBUG [arctic.store.version_store] [Thread-19 46007-47907637193024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,378 DEBUG [arctic.store.version_store] [Thread-4 46007-47907114973504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,378 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,378 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,382 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,383 INFO [arctic.arctic] [Thread-3 46007-47907104479552] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,389 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,405 DEBUG [arctic.store.version_store] [Thread-3 46007-47907104479552] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,406 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,406 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,408 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,410 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,411 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,428 INFO [arctic.arctic] [Thread-15 46007-47907508160832] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,435 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,436 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,438 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,448 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,452 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,457 DEBUG [root] [Thread-15 46007-47907508160832] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,462 DEBUG [root] [Thread-15 46007-47907508160832] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,466 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,469 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,471 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,472 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,482 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,482 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,496 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,498 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,505 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,509 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,515 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,516 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,516 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,524 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,525 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,525 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,532 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,533 DEBUG [root] [Thread-15 46007-47907508160832] auth.authenticate <42> - Auth Error localhost:27888: [Errno 32] Broken pipe 2015-12-18 14:13:09,541 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,543 INFO [arctic.arctic] [Thread-16 46007-47907561662784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,550 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,553 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,553 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,554 INFO [arctic.arctic] [Thread-19 46007-47907637193024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,556 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,557 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,561 DEBUG [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - _conn [localhost:27888: [Errno 32] Broken pipe], retrying 1 2015-12-18 14:13:09,563 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,566 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,567 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,570 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,572 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,574 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,576 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,577 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,578 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,580 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,581 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,586 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,586 DEBUG [arctic.store.version_store] [Thread-16 46007-47907561662784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,588 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,589 INFO [arctic.arctic] [Thread-3 46007-47907104479552] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,597 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,598 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,603 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,607 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,612 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,620 DEBUG [root] [Thread-19 46007-47907637193024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,627 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,637 DEBUG [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 1 2015-12-18 14:13:09,649 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,656 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,661 DEBUG [root] [Thread-19 46007-47907637193024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,663 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,667 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:09,673 DEBUG [root] [Thread-3 46007-47907104479552] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,679 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,681 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,685 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,687 DEBUG [root] [Thread-15 46007-47907508160832] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,687 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,691 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,692 DEBUG [root] [Thread-3 46007-47907104479552] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,710 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,712 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,714 DEBUG [root] [Thread-15 46007-47907508160832] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,715 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <66> - Stopping for: 6.52656590702 2015-12-18 14:13:09,718 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,721 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,723 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,729 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:09,731 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,736 DEBUG [root] [Thread-19 46007-47907637193024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:09,739 DEBUG [root] [Thread-19 46007-47907637193024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:09,755 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(16) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/16 -> socket:[26011476] 2015-12-18 14:13:09,785 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,789 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,791 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:09,832 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:09,835 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:09,836 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:10,269 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(16) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/16 -> socket:[26011509] 2015-12-18 14:13:10,784 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(35) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/35 -> socket:[26011495] 2015-12-18 14:13:11,298 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(33) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/33 -> socket:[26011494] 2015-12-18 14:13:11,812 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(36) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/36 -> socket:[26011490] 2015-12-18 14:13:12,327 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(14) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/14 -> socket:[26011452] 2015-12-18 14:13:12,842 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(26) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/26 -> socket:[26011460] 2015-12-18 14:13:13,357 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(16) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/16 -> socket:[26011566] 2015-12-18 14:13:13,872 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(7) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/7 -> socket:[26011197] 2015-12-18 14:13:14,387 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(20) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/20 -> socket:[26011480] 2015-12-18 14:13:14,902 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(41) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/41 -> socket:[26011428] 2015-12-18 14:13:15,417 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(45) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/45 -> socket:[26011436] 2015-12-18 14:13:15,932 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(32) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/32 -> socket:[26011470] 2015-12-18 14:13:16,256 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <69> - Running for: 1.46544865119 2015-12-18 14:13:16,261 DEBUG [arctic.decorators] [Thread-17 46007-47907583691072] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:16,264 DEBUG [arctic.decorators] [Thread-10 46007-47907327723840] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:16,266 DEBUG [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 2 2015-12-18 14:13:16,267 DEBUG [arctic.decorators] [Thread-1 46007-47907083491648] decorators._handle_error <88> - get_library_metadata [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:16,298 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,302 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,305 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <42> - Auth Error localhost:27888: [Errno 9] Bad file descriptor 2015-12-18 14:13:16,307 DEBUG [arctic.decorators] [Thread-12 46007-47907402209600] decorators._handle_error <88> - _auth [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:16,311 DEBUG [root] [Thread-15 46007-47907508160832] auth.authenticate <42> - Auth Error localhost:27888: [Errno 9] Bad file descriptor 2015-12-18 14:13:16,312 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,313 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,315 DEBUG [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - _auth [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:16,318 DEBUG [root] [Thread-3 46007-47907104479552] auth.authenticate <42> - Auth Error localhost:27888: [Errno 9] Bad file descriptor 2015-12-18 14:13:16,323 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,325 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,328 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,330 DEBUG [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - _conn [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:16,375 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,376 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,384 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,388 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,402 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,410 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,412 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 3 2015-12-18 14:13:16,419 DEBUG [root] [Thread-15 46007-47907508160832] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,424 DEBUG [root] [Thread-15 46007-47907508160832] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,436 DEBUG [root] [Thread-3 46007-47907104479552] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,456 DEBUG [root] [Thread-3 46007-47907104479552] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,503 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,519 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,523 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,540 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,561 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,569 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,579 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(30) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/30 -> socket:[26011484] 2015-12-18 14:13:16,584 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,587 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:16,587 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,598 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:16,604 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,632 INFO [arctic.arctic] [Thread-19 46007-47907637193024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,634 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:16,640 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,642 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:16,644 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,653 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,661 DEBUG [arctic.store.version_store] [Thread-19 46007-47907637193024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,699 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,705 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,712 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,735 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 4 2015-12-18 14:13:16,740 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,743 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,762 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,765 DEBUG [arctic.store.version_store] [Thread-11 46007-47907348707648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,769 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,779 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,792 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,798 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,798 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,809 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:16,811 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:16,823 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,826 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:16,831 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:16,837 INFO [arctic.arctic] [Thread-19 46007-47907637193024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,839 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:16,844 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,846 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,847 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:16,851 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,867 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,868 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,868 DEBUG [root] [Thread-19 46007-47907637193024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:16,872 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,875 DEBUG [root] [Thread-19 46007-47907637193024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:16,875 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,880 INFO [arctic.arctic] [Thread-15 46007-47907508160832] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,884 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,885 INFO [arctic.arctic] [Thread-3 46007-47907104479552] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,899 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,905 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,909 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,913 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:16,920 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:16,925 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:16,929 DEBUG [arctic.store.version_store] [Thread-15 46007-47907508160832] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,938 DEBUG [arctic.store.version_store] [Thread-3 46007-47907104479552] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,938 DEBUG [arctic.store.version_store] [Thread-4 46007-47907114973504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:16,946 DEBUG [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 1 2015-12-18 14:13:16,948 DEBUG [root] [Thread-19 46007-47907637193024] auth.authenticate <42> - Auth Error localhost:27888: [Errno 32] Broken pipe 2015-12-18 14:13:16,959 DEBUG [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 1 2015-12-18 14:13:16,961 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:16,969 DEBUG [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - _conn [localhost:27888: [Errno 32] Broken pipe], retrying 1 2015-12-18 14:13:16,977 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,981 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:16,985 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:16,994 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,000 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,005 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,008 DEBUG [arctic.store.version_store] [Thread-17 46007-47907583691072] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,013 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,015 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,016 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,019 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,033 DEBUG [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 2 2015-12-18 14:13:17,039 DEBUG [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 2 2015-12-18 14:13:17,040 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,041 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,047 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,053 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,056 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,057 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,063 DEBUG [root] [Thread-19 46007-47907637193024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,067 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,071 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,075 DEBUG [root] [Thread-19 46007-47907637193024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,086 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,087 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,092 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,100 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,101 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,107 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,108 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,107 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,112 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,114 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,135 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,147 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,149 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,156 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,157 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 3 2015-12-18 14:13:17,160 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,161 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,186 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,188 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,191 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 3 2015-12-18 14:13:17,209 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,215 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,219 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,224 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,225 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,230 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,248 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,253 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,255 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,256 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,256 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,259 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,262 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(26) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/26 -> socket:[26011783] 2015-12-18 14:13:17,275 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,284 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,286 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 5 2015-12-18 14:13:17,300 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,307 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,311 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,316 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,317 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,318 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,321 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,323 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,345 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,361 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,365 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,366 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,366 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,368 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,372 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,374 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,374 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,377 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,383 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,392 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,424 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,436 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,438 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,445 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,465 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,467 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [localhost:27888: [Errno 9] Bad file descriptor], retrying 4 2015-12-18 14:13:17,470 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,471 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,475 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,481 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,481 INFO [arctic.arctic] [Thread-19 46007-47907637193024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,486 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,494 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,499 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,516 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,518 DEBUG [arctic.store.version_store] [Thread-19 46007-47907637193024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,520 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,521 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,525 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,526 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 4 2015-12-18 14:13:17,528 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,533 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,542 DEBUG [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 1 2015-12-18 14:13:17,544 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,566 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,568 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,570 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,575 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,582 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,590 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,591 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,599 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,604 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,618 DEBUG [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 2 2015-12-18 14:13:17,624 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,627 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,636 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,639 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,640 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,647 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,668 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,684 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,686 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:17,689 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,690 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,694 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:17,706 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,710 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:17,714 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,726 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,727 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <66> - Stopping for: 4.7512268565 2015-12-18 14:13:17,734 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:17,737 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,739 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,743 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,745 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,747 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,747 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,748 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,782 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:17,791 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:17,794 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:17,803 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(50) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/50 -> socket:[26011438] 2015-12-18 14:13:18,839 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(10) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/10 -> socket:[26011444] 2015-12-18 14:13:19,357 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(25) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/25 -> socket:[26012283] 2015-12-18 14:13:19,874 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(76) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/76 -> socket:[26012063] 2015-12-18 14:13:20,909 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(38) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/38 -> socket:[26011409] 2015-12-18 14:13:21,427 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(78) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/78 -> socket:[26012273] 2015-12-18 14:13:21,945 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(24) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/24 -> socket:[26012450] 2015-12-18 14:13:22,461 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(64) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/64 -> socket:[26011861] 2015-12-18 14:13:22,485 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <69> - Running for: 2.84822874377 2015-12-18 14:13:22,486 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,488 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 3 2015-12-18 14:13:22,491 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 6 2015-12-18 14:13:22,492 DEBUG [arctic.decorators] [Thread-7 46007-47907146455360] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:22,493 DEBUG [arctic.decorators] [Thread-4 46007-47907114973504] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:22,506 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 5 2015-12-18 14:13:22,527 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,529 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,532 DEBUG [arctic.store.version_store] [Thread-17 46007-47907583691072] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,535 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,547 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,549 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,569 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,608 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,621 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,636 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,642 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,647 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,654 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,655 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,667 DEBUG [arctic.store.version_store] [Thread-11 46007-47907348707648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,668 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,669 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,674 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,691 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,692 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,696 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,704 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,710 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,714 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,715 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,717 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,723 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,728 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,739 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,741 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,743 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,748 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,749 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,753 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,757 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,760 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,766 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:22,785 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,789 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,792 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,792 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,794 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,799 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,801 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,804 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,809 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,811 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,819 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 4 2015-12-18 14:13:22,839 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,847 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,853 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,857 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,859 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,860 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,862 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,865 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,868 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,870 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,873 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,875 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,893 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,932 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,936 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,937 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,940 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,940 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:22,940 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:22,941 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,943 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:22,945 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,946 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,970 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,974 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:22,981 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:22,981 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:22,995 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,009 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,010 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,011 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,018 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,023 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 5 2015-12-18 14:13:23,024 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,033 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,045 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,055 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,067 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,068 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,069 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,079 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,083 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 6 2015-12-18 14:13:23,084 DEBUG [arctic.store.version_store] [Thread-17 46007-47907583691072] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,089 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,107 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,120 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,172 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,182 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,183 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,193 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(77) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/77 -> socket:[26011957] 2015-12-18 14:13:23,210 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,212 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,214 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,219 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,224 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,243 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,249 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,251 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,272 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,275 DEBUG [arctic.store.version_store] [Thread-4 46007-47907114973504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,276 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,297 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,298 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,301 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,309 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,317 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,321 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,322 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,334 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,347 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,351 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,365 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 5 2015-12-18 14:13:23,369 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,377 DEBUG [arctic.store.version_store] [Thread-11 46007-47907348707648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,377 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,384 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,383 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,385 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,391 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,392 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,397 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,399 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,401 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,414 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,422 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,425 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,426 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,427 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,433 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,434 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,439 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,444 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,451 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,454 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,454 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,455 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,464 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,471 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,478 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,482 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,487 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,488 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,493 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,494 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,502 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,510 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,515 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,521 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,524 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,533 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,548 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,550 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,553 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,561 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,564 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,571 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 7 2015-12-18 14:13:23,589 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,591 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,599 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,600 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,601 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 6 2015-12-18 14:13:23,604 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,607 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,618 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,635 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,662 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,663 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,668 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,669 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,670 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,670 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 7 2015-12-18 14:13:23,686 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,690 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,694 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,701 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,715 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,721 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,722 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,724 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,731 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,749 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,753 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,757 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,758 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,761 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,762 DEBUG [arctic.store.version_store] [Thread-17 46007-47907583691072] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,762 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,764 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,767 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,769 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,772 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,774 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,777 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,778 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,785 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,790 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,802 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,805 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,807 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,810 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,811 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,816 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,828 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,834 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,851 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,856 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,859 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,878 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,882 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(23) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/23 -> socket:[26011458] 2015-12-18 14:13:23,902 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:23,902 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,913 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 6 2015-12-18 14:13:23,914 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,919 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,921 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,936 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:23,945 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:23,957 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:23,961 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:23,966 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:23,969 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:23,980 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,000 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,006 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,006 DEBUG [arctic.store.version_store] [Thread-11 46007-47907348707648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,009 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,009 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,016 DEBUG [arctic.store.version_store] [Thread-4 46007-47907114973504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,025 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,034 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,043 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,046 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,062 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,064 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,071 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,071 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,075 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,083 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,091 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,096 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,098 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,100 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,105 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,111 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,120 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,121 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 8 2015-12-18 14:13:24,119 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,123 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,129 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,134 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,137 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,143 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,146 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,147 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,149 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,153 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 7 2015-12-18 14:13:24,156 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,184 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,187 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,190 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,190 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,192 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,193 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,200 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,207 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,223 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,224 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 8 2015-12-18 14:13:24,228 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,234 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,236 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,242 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,244 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,245 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,256 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,261 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,262 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,270 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,295 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,299 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,302 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,302 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,304 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,311 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,311 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,317 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,321 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,324 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,326 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,331 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,332 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,347 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,349 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,353 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,361 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,363 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,368 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,371 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,374 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,376 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,385 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,386 DEBUG [arctic.store.version_store] [Thread-17 46007-47907583691072] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,391 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,399 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,410 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,414 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,419 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,434 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,442 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,444 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,450 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,452 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,455 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,472 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 7 2015-12-18 14:13:24,477 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,483 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,498 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,498 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,500 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,501 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,505 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,506 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,514 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,552 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,559 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,563 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,585 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,593 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,595 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,604 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(45) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/45 -> socket:[26012749] 2015-12-18 14:13:24,616 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,617 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,621 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,633 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,641 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,645 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,664 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,668 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 9 2015-12-18 14:13:24,672 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,674 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,683 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,686 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,686 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,689 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,690 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,702 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,703 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,707 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 8 2015-12-18 14:13:24,730 DEBUG [arctic.store.version_store] [Thread-11 46007-47907348707648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,732 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,743 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,746 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,746 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,748 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,753 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,754 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,756 DEBUG [arctic.store.version_store] [Thread-4 46007-47907114973504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,758 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,758 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,766 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 9 2015-12-18 14:13:24,770 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,778 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,781 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,786 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,790 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,794 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,806 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,810 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,812 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,817 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,823 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,827 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,830 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,831 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,835 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,835 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,855 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,874 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,877 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,881 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,882 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,882 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,888 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,895 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,898 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,903 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,916 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,922 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,933 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,934 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,935 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,935 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,939 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,952 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:24,965 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:24,968 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:24,970 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:24,976 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,978 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,979 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:24,982 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:24,985 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:24,991 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,003 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,007 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,010 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 8 2015-12-18 14:13:25,015 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:25,023 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,025 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,025 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,030 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:25,036 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,050 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,055 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,057 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,058 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,059 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,064 DEBUG [arctic.store.version_store] [Thread-17 46007-47907583691072] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:25,069 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,073 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,077 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,082 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,093 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,098 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,100 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,107 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:25,118 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,119 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:25,120 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,133 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,153 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:25,156 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,161 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:25,162 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,176 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,178 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,184 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,187 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,200 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:25,202 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,205 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:25,210 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 10 2015-12-18 14:13:25,240 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:25,244 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:25,252 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,257 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,258 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 9 2015-12-18 14:13:25,263 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,265 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,273 INFO [arctic.arctic] [Thread-17 46007-47907583691072] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,331 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,336 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <66> - Stopping for: 2.84077944048 ls: /proc/46007/fd/29: No such file or directory 2015-12-18 14:13:25,339 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 10 2015-12-18 14:13:25,343 ERROR [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <90> - Failed to Close Traceback (most recent call last): File "/users/is/jblackburn/workspaces/scratch/mongo/MDP-341/chaos_monkey.py", line 85, in chaos_monkey status = subprocess.check_output(['ls', '-l', BASE_PATH + str(fd)]) File "/opt/ahl/releases/python/2.7.3/lib64/python2.7/subprocess.py", line 544, in check_output raise CalledProcessError(retcode, cmd, output=output) CalledProcessError: Command '['ls', '-l', '/proc/46007/fd/29']' returned non-zero exit status 2 2015-12-18 14:13:25,345 DEBUG [root] [Thread-17 46007-47907583691072] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,347 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:25,347 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,350 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,352 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,353 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:25,355 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,358 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,359 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,373 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,373 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:25,379 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,380 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:25,381 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,382 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:25,873 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(132) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/132 -> socket:[26012827] 2015-12-18 14:13:26,393 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(69) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/69 -> socket:[26011869] 2015-12-18 14:13:26,914 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(16) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/16 -> socket:[26012692] 2015-12-18 14:13:27,437 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(102) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/102 -> socket:[26012664] 2015-12-18 14:13:27,960 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(124) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/124 -> socket:[26012817] 2015-12-18 14:13:28,179 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <69> - Running for: 1.08864632967 2015-12-18 14:13:28,180 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 11 2015-12-18 14:13:28,182 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [localhost:27888: [Errno 9] Bad file descriptor], retrying 9 2015-12-18 14:13:28,183 DEBUG [arctic.decorators] [Thread-11 46007-47907348707648] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:28,190 DEBUG [arctic.decorators] [Thread-14 46007-47907476691264] decorators._handle_error <88> - get_library_metadata [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:28,194 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 11 2015-12-18 14:13:28,194 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 10 2015-12-18 14:13:28,199 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,214 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,231 DEBUG [root] [Thread-17 46007-47907583691072] auth.authenticate <42> - Auth Error wire protocol error: unknown opcode 0 2015-12-18 14:13:28,244 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,246 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,247 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,250 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,262 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,264 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,264 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,267 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,274 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,278 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,280 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,290 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,294 DEBUG [arctic.store.version_store] [Thread-4 46007-47907114973504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,311 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,318 ERROR [root] [Thread-17 46007-47907583691072] chaos_monkey.run_forever <54> - Failed run_forever None 2015-12-18 14:13:28,329 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used Exception in thread Thread-17: Traceback (most recent call last): File "/opt/ahl/releases/python/2.7.3/lib64/python2.7/threading.py", line 551, in __bootstrap_inner self.run() File "/opt/ahl/releases/python/2.7.3/lib64/python2.7/threading.py", line 504, in run self.__target(*self.__args, **self.__kwargs) File "/users/is/jblackburn/workspaces/scratch/mongo/MDP-341/chaos_monkey.py", line 48, in run_forever connect_and_write() File "/users/is/jblackburn/workspaces/scratch/mongo/MDP-341/chaos_monkey.py", line 35, in connect_and_write lib = m['jblackburn.scratch'] File "/users/is/jblackburn/git/ahl.mongo/ahl/mongo/mongoose/mongoose.py", line 151, in __getitem__ return self.get_library(key) File "/users/is/jblackburn/git/ahl.mongo/ahl/mongo/decorators.py", line 64, in f_retry return arctic.decorators.mongo_retry(f)(*args, **kwargs) File "/users/is/jblackburn/git/arctic/arctic/decorators.py", line 53, in f_retry return f(*args, **kwargs) File "/users/is/jblackburn/git/ahl.mongo/ahl/mongo/mongoose/mongoose.py", line 129, in get_library l = MongooseLibrary(self, library) File "/users/is/jblackburn/git/arctic/arctic/arctic.py", line 341, in __init__ self._db = self.arctic._conn[database_name] File "/users/is/jblackburn/git/arctic/arctic/decorators.py", line 53, in f_retry return f(*args, **kwargs) File "/users/is/jblackburn/git/arctic/arctic/arctic.py", line 121, in _conn authenticate(self._adminDB, auth.user, auth.password) File "", line 2, in authenticate File "/users/is/jblackburn/pyenvs/research/lib/python2.7/site-packages/ahl.diags_core-1.65.0.dev1-py2.7.egg/ahl/diagscore/events.py", line 320, in event return f(*args, **kwargs) File "/users/is/jblackburn/git/ahl.mongo/ahl/mongo/auth.py", line 28, in authenticate res = db.authenticate(user, password) File "/users/is/jblackburn/git/mongo-python-driver/pymongo/database.py", line 1023, in authenticate connect=True) File "/users/is/jblackburn/git/mongo-python-driver/pymongo/mongo_client.py", line 422, in _cache_credentials with server.get_socket(all_credentials) as sock_info: File "/opt/ahl/releases/python/2.7.3/lib64/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/users/is/jblackburn/git/mongo-python-driver/pymongo/server.py", line 152, in get_socket with self.pool.get_socket(all_credentials, checkout) as sock_info: File "/opt/ahl/releases/python/2.7.3/lib64/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/users/is/jblackburn/git/mongo-python-driver/pymongo/pool.py", line 536, in get_socket sock_info = self._get_socket_no_auth() File "/users/is/jblackburn/git/mongo-python-driver/pymongo/pool.py", line 570, in _get_socket_no_auth sock_info, from_pool = self.connect(), False File "/users/is/jblackburn/git/mongo-python-driver/pymongo/pool.py", line 502, in connect DEFAULT_CODEC_OPTIONS)) File "/users/is/jblackburn/git/mongo-python-driver/pymongo/network.py", line 92, in command response = receive_message(sock, 1, request_id) File "/users/is/jblackburn/git/mongo-python-driver/pymongo/network.py", line 125, in receive_message "unknown opcode %r" % (actual_op,)) AssertionError: wire protocol error: unknown opcode 0 2015-12-18 14:13:28,349 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,352 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,359 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,365 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,375 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,391 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,398 INFO [root] [MainThread 46007-47906685347328] chaos_monkey. <110> - calling sys.exit 2015-12-18 14:13:28,407 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,408 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,411 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,416 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,420 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,433 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,441 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,443 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,446 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,451 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,452 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,464 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,467 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,483 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,483 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,483 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,485 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,489 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,490 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,493 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,503 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,505 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,510 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,514 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,517 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,523 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,526 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,527 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:28,527 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,559 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,559 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(94) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/94 -> socket:[26012652] 2015-12-18 14:13:28,560 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,562 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,568 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,571 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,576 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,579 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,580 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,583 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,587 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,592 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,606 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,607 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,614 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,615 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,618 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,620 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,623 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,623 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,629 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,625 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,639 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,648 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,649 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,657 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,676 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,684 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,687 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,697 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,709 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,713 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,715 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,728 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,745 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 12 2015-12-18 14:13:28,748 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,751 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,754 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,775 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 10 2015-12-18 14:13:28,776 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,780 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 11 2015-12-18 14:13:28,779 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,783 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 12 2015-12-18 14:13:28,804 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,806 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,815 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,817 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,820 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,830 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,839 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,843 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,854 DEBUG [arctic.store.version_store] [Thread-4 46007-47907114973504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,858 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,863 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,869 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,876 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,885 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,891 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,897 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,906 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,907 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:28,908 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,922 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,924 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,929 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,939 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,949 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,951 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:28,960 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:28,968 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,979 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:28,986 DEBUG [arctic.store.version_store] [Thread-6 46007-47907135961408] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,991 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:28,993 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:28,996 DEBUG [arctic.store.version_store] [Thread-11 46007-47907348707648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:28,997 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:29,016 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,021 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:29,022 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:29,028 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,040 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,042 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:29,045 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:29,048 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,053 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,056 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,059 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,063 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,073 DEBUG [arctic.store.version_store] [Thread-7 46007-47907146455360] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:29,091 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:29,091 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:29,092 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,094 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,095 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:29,097 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:29,099 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,100 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,111 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,112 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:29,123 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,124 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,127 DEBUG [arctic.store.version_store] [Thread-12 46007-47907402209600] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:29,129 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,131 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:29,132 INFO [arctic.arctic] [Thread-6 46007-47907135961408] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,146 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:29,150 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:29,161 DEBUG [arctic.store.version_store] [Thread-2 46007-47907093981504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:29,163 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:29,164 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,171 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:29,175 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,176 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,176 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,177 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:29,178 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:29,182 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,186 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:29,192 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,206 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,206 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:29,214 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:29,218 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,221 DEBUG [arctic.store.version_store] [Thread-1 46007-47907083491648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:29,225 DEBUG [root] [Thread-6 46007-47907135961408] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:29,228 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,229 DEBUG [root] [Thread-6 46007-47907135961408] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:29,230 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,239 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,257 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:29,267 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,271 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <66> - Stopping for: 1.31563435774 2015-12-18 14:13:29,272 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:29,280 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:29,280 INFO [arctic.arctic] [Thread-7 46007-47907146455360] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,281 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(120) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/120 -> socket:[26012807] 2015-12-18 14:13:29,283 INFO [arctic.arctic] [Thread-12 46007-47907402209600] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,288 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,289 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,290 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,291 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,292 INFO [arctic.arctic] [Thread-2 46007-47907093981504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,296 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,297 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:29,358 INFO [arctic.arctic] [Thread-1 46007-47907083491648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:29,361 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:29,363 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:29,794 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(114) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/114 -> socket:[26012686] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:30,306 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(11) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/11 -> socket:[26011446] 2015-12-18 14:13:30,590 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <69> - Running for: 0.188690049917 2015-12-18 14:13:30,597 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 13 2015-12-18 14:13:30,597 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 13 2015-12-18 14:13:30,602 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 12 2015-12-18 14:13:30,603 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 11 2015-12-18 14:13:30,610 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:30,620 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:30,627 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:30,642 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:30,644 DEBUG [root] [Thread-7 46007-47907146455360] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:30,652 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:30,653 DEBUG [root] [Thread-7 46007-47907146455360] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:30,666 DEBUG [root] [Thread-12 46007-47907402209600] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:30,675 DEBUG [root] [Thread-12 46007-47907402209600] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:30,681 DEBUG [root] [Thread-2 46007-47907093981504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:30,685 DEBUG [root] [Thread-2 46007-47907093981504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:30,689 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:30,692 DEBUG [root] [Thread-1 46007-47907083491648] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:30,700 DEBUG [root] [Thread-1 46007-47907083491648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:30,768 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:30,768 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:30,775 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:30,780 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <66> - Stopping for: 5.91564846063 2015-12-18 14:13:30,786 DEBUG [arctic.store.version_store] [Thread-14 46007-47907476691264] version_store.write <570> - Finished writing versions for foo Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:30,819 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(114) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/114 -> socket:[26013329] 2015-12-18 14:13:30,843 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:30,846 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:30,848 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:31,329 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(73) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/73 -> socket:[26011876] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:31,851 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(137) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/137 -> socket:[26012841] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:32,364 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(95) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/95 -> socket:[26012649] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:32,875 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(155) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/155 -> socket:[26013274] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:33,386 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(105) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/105 -> socket:[26012670] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:33,897 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(4) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/4 -> socket:[26012456] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:34,408 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(146) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/146 -> socket:[26013256] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:34,918 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(84) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/84 -> socket:[26013228] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:35,428 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(36) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/36 -> socket:[26013170] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:35,938 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(26) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/26 -> socket:[26013175] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:36,448 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(45) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/45 -> socket:[26013200] 2015-12-18 14:13:36,707 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <69> - Running for: 0.300965071944 2015-12-18 14:13:36,709 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 14 2015-12-18 14:13:36,712 DEBUG [arctic.decorators] [Thread-2 46007-47907093981504] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:36,713 DEBUG [arctic.store.version_store] [Thread-5 46007-47907125467456] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,716 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 12 2015-12-18 14:13:36,718 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 13 2015-12-18 14:13:36,719 DEBUG [arctic.decorators] [Thread-6 46007-47907135961408] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:36,722 DEBUG [arctic.store.version_store] [Thread-4 46007-47907114973504] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,723 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 14 2015-12-18 14:13:36,755 INFO [root] [Thread-5 46007-47907125467456] chaos_monkey.run_forever <51> - writing: 10 2015-12-18 14:13:36,761 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:36,767 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:36,776 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:36,777 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:36,782 DEBUG [arctic.store.version_store] [Thread-20 46007-47907658172736] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,790 INFO [root] [Thread-20 46007-47907658172736] chaos_monkey.run_forever <51> - writing: 10 2015-12-18 14:13:36,802 DEBUG [arctic.store.version_store] [Thread-13 46007-47907444169024] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,812 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:36,817 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:36,820 INFO [root] [Thread-13 46007-47907444169024] chaos_monkey.run_forever <51> - writing: 10 2015-12-18 14:13:36,826 DEBUG [arctic.store.version_store] [Thread-9 46007-47907283667264] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,832 DEBUG [arctic.store.version_store] [Thread-8 46007-47907178142016] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,833 INFO [root] [Thread-9 46007-47907283667264] chaos_monkey.run_forever <51> - writing: 10 2015-12-18 14:13:36,837 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:36,839 INFO [root] [Thread-8 46007-47907178142016] chaos_monkey.run_forever <51> - writing: 10 2015-12-18 14:13:36,843 DEBUG [arctic.store.version_store] [Thread-10 46007-47907327723840] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,847 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:36,850 INFO [root] [Thread-10 46007-47907327723840] chaos_monkey.run_forever <51> - writing: 10 2015-12-18 14:13:36,853 DEBUG [arctic.store.version_store] [Thread-11 46007-47907348707648] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,853 INFO [arctic.arctic] [Thread-14 46007-47907476691264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:36,862 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:36,865 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:36,867 INFO [arctic.arctic] [Thread-5 46007-47907125467456] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:36,874 INFO [arctic.arctic] [Thread-4 46007-47907114973504] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:36,879 DEBUG [root] [Thread-14 46007-47907476691264] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:36,879 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:36,881 DEBUG [root] [Thread-14 46007-47907476691264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:36,883 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:36,883 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:36,888 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:36,894 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic.check_quota <435> - Mongo Quota: 0.000 / 10 GB used 2015-12-18 14:13:36,898 DEBUG [root] [Thread-5 46007-47907125467456] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:36,900 DEBUG [root] [Thread-5 46007-47907125467456] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:36,904 DEBUG [root] [Thread-4 46007-47907114973504] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:36,906 DEBUG [root] [Thread-4 46007-47907114973504] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:36,906 DEBUG [arctic.store.version_store] [Thread-18 46007-47907604670784] version_store.write <570> - Finished writing versions for foo 2015-12-18 14:13:36,914 INFO [arctic.arctic] [Thread-20 46007-47907658172736] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:36,922 INFO [root] [Thread-18 46007-47907604670784] chaos_monkey.run_forever <51> - writing: 10 2015-12-18 14:13:36,928 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:36,929 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:36,934 INFO [arctic.arctic] [Thread-13 46007-47907444169024] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:36,946 DEBUG [root] [Thread-20 46007-47907658172736] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:36,951 DEBUG [root] [Thread-20 46007-47907658172736] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:36,951 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:36,956 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:36,975 DEBUG [root] [Thread-13 46007-47907444169024] mongo.get_auth <49> - get_auth('localhost:27888', 'mongoose', 'mongoose_jblackburn') => ('mongoose_jblackburn', 'jblackburn', ...) 2015-12-18 14:13:36,979 DEBUG [root] [Thread-13 46007-47907444169024] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'mongoose_jblackburn') with jblackburn 2015-12-18 14:13:36,989 INFO [arctic.arctic] [Thread-8 46007-47907178142016] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:37,009 INFO [arctic.arctic] [Thread-11 46007-47907348707648] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:37,015 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <66> - Stopping for: 7.57896620623 Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:37,032 DEBUG [root] [Thread-8 46007-47907178142016] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:37,032 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(33) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/33 -> socket:[26013166] 2015-12-18 14:13:37,035 DEBUG [root] [Thread-8 46007-47907178142016] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:37,036 DEBUG [root] [Thread-11 46007-47907348707648] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:37,036 INFO [arctic.arctic] [Thread-9 46007-47907283667264] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:37,037 INFO [arctic.arctic] [Thread-18 46007-47907604670784] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:37,040 DEBUG [root] [Thread-11 46007-47907348707648] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:37,042 INFO [arctic.arctic] [Thread-10 46007-47907327723840] arctic._conn <110> - Connecting to mongo: localhost:27888 (localhost:27888) 2015-12-18 14:13:37,056 DEBUG [root] [Thread-9 46007-47907283667264] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:37,058 DEBUG [root] [Thread-9 46007-47907283667264] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:37,058 DEBUG [root] [Thread-18 46007-47907604670784] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:37,059 DEBUG [root] [Thread-18 46007-47907604670784] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research 2015-12-18 14:13:37,060 DEBUG [root] [Thread-10 46007-47907327723840] mongo.get_auth <56> - get_auth('localhost:27888', 'mongoose', 'admin') => ('*', 'research', ...) 2015-12-18 14:13:37,061 DEBUG [root] [Thread-10 46007-47907327723840] auth.authenticate <27> - Authenticating Database(MongoClient(host=['localhost:27888'], document_class=dict, tz_aware=False, connect=True, maxpoolsize=4, serverselectiontimeoutms='30000', connecttimeoutms='2000', sockettimeoutms='600000'), u'admin') with research Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:38,057 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(131) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/131 -> socket:[26012825] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:38,568 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(69) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/69 -> socket:[26013230] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:39,589 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(96) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/96 -> socket:[26012662] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:40,100 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(135) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/135 -> socket:[26012835] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:40,612 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(9) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/9 -> socket:[26012468] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:41,123 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(71) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/71 -> socket:[26011943] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:41,633 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(102) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/102 -> socket:[26013236] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:42,144 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(9) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/9 -> socket:[26013876] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:42,655 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(4) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/4 -> socket:[26013902] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:43,164 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(91) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/91 -> socket:[26012647] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:43,675 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(6) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/6 -> socket:[26012454] Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in ignored 2015-12-18 14:13:44,185 INFO [root] [Thread-22 46007-47907700136256] chaos_monkey.chaos_monkey <88> - CLOSE'ing(133) lrwx------ 1 jblackburn is 64 Dec 18 14:13 /proc/46007/fd/133 -> socket:[26012829] 2015-12-18 14:13:44,603 INFO [root] [Thread-21 46007-47907689646400] chaos_monkey.pause_mongo <69> - Running for: 2.6090363098 2015-12-18 14:13:44,606 DEBUG [arctic.decorators] [Thread-14 46007-47907476691264] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:44,608 DEBUG [arctic.decorators] [Thread-13 46007-47907444169024] decorators._handle_error <88> - _ensure_index [localhost:27888: [Errno 9] Bad file descriptor], retrying 1 2015-12-18 14:13:44,608 WARNI [arctic.decorators] [Thread-19 46007-47907637193024] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 13 2015-12-18 14:13:44,613 WARNI [arctic.decorators] [Thread-15 46007-47907508160832] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 14 2015-12-18 14:13:44,616 WARNI [arctic.decorators] [Thread-3 46007-47907104479552] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 15 2015-12-18 14:13:44,620 WARNI [arctic.decorators] [Thread-16 46007-47907561662784] decorators._handle_error <88> - list_libraries [command SON([('listDatabases', 1)]) on namespace admin.$cmd failed: unauthorized], retrying 15