Leveldb corruption repair on restart or shutdown

[jagguli]

Currently I have this hack to fix leveldb corruptions

#!/usr/bin/env python
import os
import re
from datetime import datetime
leveldb_pattern = re.compile(
    "^.*Failed to start riak_kv_multi_backend backend for index (\d*) error:.*$"
)
leveldb_pattern1 = re.compile(
    "^.*state,548063113999088594326381812268606132370974703616,riak_kv_multi_backend,false,.*$"
)

yz_aae_pattern0 = re.compile(
    r"^.*When Server state == \{state,automatic,\[\{(\d*),.*$"
)

#2020-05-28 03:16:43.103 [info] <0.1858.0>@riak_kv_vnode:maybe_create_hashtrees:275 riak_kv/365375409332725729550921208179070754913983135744: unable to start index_hashtree:
yz_aae_pattern1 = re.compile("^.*riak_kv_vnode:maybe_create_hashtrees:275 riak_kv/(\d*): unable to start index_hashtree:.*")

bitcask_pattern = re.compile(r'.*bc_state,"/var/lib/riak/bitcask/(\d*)",fresh.*')
LOG_FILE = "/var/log/riak/console.log"
REPAIR_CMD = "/usr/lib/riak/lib/eleveldb-riak_kv-2.9.2-0-gc7018b6/priv/leveldb_repair"
repaired = set([])
for i, line in enumerate(open(LOG_FILE)):
    for filepath, pattern in [
        ("/var/lib/riak/leveldb*/", leveldb_pattern),
        ("/var/lib/riak/leveldb*/", leveldb_pattern1),
        ("/var/lib/riak/anti_entropy/", yz_aae_pattern0),
        ("/var/lib/riak/yz_anti_entropy/", yz_aae_pattern1),
    ]:
        match = re.match(pattern, line)
        if match:
            if (filepath, match.group()[0]) in repaired:
                continue
            print('Found corruption on line %s: %s' % (i+1, match.group()))
            os.system("%s %s%s" % (REPAIR_CMD, filepath, match.groups()[0]))
            repaired.add((filepath, match.group()[0]))
os.system("mv %s %s.%s" % (LOG_FILE, LOG_FILE, datetime.utcnow().strftime("%Y%m%d%H%M")))