As reported by me on wiredtiger-users list:

Attempt to enable transaction logging with custom collator led to classic catch-22 situation.
Demonstration test:

$ cat b.c

include <assert.h>

include <stdio.h>

include <string.h>

include <wiredtiger.h>

define DB_HOME "coll.db"

static int
my_compare(WT_COLLATOR _collator, WT_SESSION _session, const WT_ITEM *v1, const WT_ITEM _v2, int _cmp)
{
uint64_t a = *(uint64_t *)v1->data;
uint64_t b = *(uint64_t *)v2->data;
*cmp = (a < b ? -1 : (a > b ? 1 : 0));
return 0;
}

static WT_COLLATOR my_coll =

{ my_compare, NULL }

;

int main(int argc, char **argv)
{
WT_CONNECTION *conn;
WT_SESSION *session;
WT_CURSOR *cursor;
uint64_t key;

if (argc > 1) {
assert(system("rm -rf " DB_HOME " && mkdir " DB_HOME) == 0);

assert(wiredtiger_open(DB_HOME, NULL, "create,log=(enabled)", &conn) == 0);
assert(conn->add_collator(conn, "my_coll", &my_coll, NULL) == 0);

assert(conn->open_session(conn, NULL, NULL, &session) == 0);
assert(session->create(session, "table:access", "key_format=Q,value_format=Q,collator=my_coll") == 0);

assert(session->open_cursor(session, "table:access", NULL, NULL, &cursor) == 0);

for (key = 0; key < 10; ++key)

{ cursor->set_key(cursor, key); cursor->set_value(cursor, key); assert(cursor->insert(cursor) == 0); }

_Exit(0);
}

assert(wiredtiger_open(DB_HOME, NULL, "log=(enabled)", &conn) == 0);
assert(conn->add_collator(conn, "my_coll", &my_coll, NULL) == 0);

assert(conn->close(conn, NULL) == 0);

return 0;
}
$ gcc -lwiredtiger b.c
$ ./a.out create-and-crash
$ ./a.out
[1410363294:40718][27478:009737a8267f0000], file:access.wt: unknown collator 'my_coll': Invalid argument
[1410363294:40760][27478:009737a8267f0000], file:access.wt: Operation failed during recovery: Invalid argument
a.out: b.c:47: main: Assertion `wiredtiger_open("coll.db", ((void *)0), "log=(enabled)", &conn) == 0' failed.
Aborted (core dumped)