diff --git a/src/config/common/vnc_cassandra.py b/src/config/common/vnc_cassandra.py index 790acd1b446..b3e7400940b 100644 --- a/src/config/common/vnc_cassandra.py +++ b/src/config/common/vnc_cassandra.py @@ -15,6 +15,8 @@ from pysandesh.gen_py.process_info.ttypes import ConnectionStatus, \ ConnectionType from pysandesh.gen_py.sandesh.ttypes import SandeshLevel +from sandesh_common.vns.constants import API_SERVER_KEYSPACE_NAME, \ + CASSANDRA_DEFAULT_GC_GRACE_SECONDS import time from cfgm_common import jsonutils as json import utils @@ -25,7 +27,7 @@ class VncCassandraClient(object): # Name to ID mapping keyspace + tables - _UUID_KEYSPACE_NAME = 'config_db_uuid' + _UUID_KEYSPACE_NAME = API_SERVER_KEYSPACE_NAME # TODO describe layout _OBJ_UUID_CF_NAME = 'obj_uuid_table' @@ -174,9 +176,7 @@ def _cassandra_ensure_keyspace(self, server_list, # TODO verify only EEXISTS self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN) - gc_grace_sec = 0 - if self._num_dbnodes > 1: - gc_grace_sec = 60 + gc_grace_sec = CASSANDRA_DEFAULT_GC_GRACE_SECONDS for cf_info in cf_info_list: try: diff --git a/src/config/schema-transformer/db.py b/src/config/schema-transformer/db.py index ae932b4cb9f..b2d93df67c0 100644 --- a/src/config/schema-transformer/db.py +++ b/src/config/schema-transformer/db.py @@ -11,10 +11,11 @@ import cfgm_common as common from cfgm_common.zkclient import IndexAllocator from cfgm_common.vnc_cassandra import VncCassandraClient +from sandesh_common.vns.constants import SCHEMA_KEYSPACE_NAME class SchemaTransformerDB(VncCassandraClient): - _KEYSPACE = 'to_bgp_keyspace' + _KEYSPACE = SCHEMA_KEYSPACE_NAME _RT_CF = 'route_target_table' _SC_IP_CF = 'service_chain_ip_address_table' _SERVICE_CHAIN_CF = 'service_chain_table' diff --git a/src/config/svc-monitor/svc_monitor/db.py b/src/config/svc-monitor/svc_monitor/db.py index 924af45daeb..a4f3bec4525 100644 --- a/src/config/svc-monitor/svc_monitor/db.py +++ b/src/config/svc-monitor/svc_monitor/db.py @@ -10,11 +10,11 @@ from cfgm_common import jsonutils as json from cfgm_common.vnc_cassandra import VncCassandraClient - +from sandesh_common.vns.constants import SVC_MONITOR_KEYSPACE_NAME class ServiceMonitorDB(VncCassandraClient): - _KEYSPACE = 'svc_monitor_keyspace' + _KEYSPACE = SVC_MONITOR_KEYSPACE_NAME _SVC_SI_CF = 'service_instance_table' _LB_CF = 'pool_table' diff --git a/src/discovery/disc_cassdb.py b/src/discovery/disc_cassdb.py index 512d2eae3cf..d55b8fb342f 100644 --- a/src/discovery/disc_cassdb.py +++ b/src/discovery/disc_cassdb.py @@ -16,9 +16,11 @@ from pycassa.system_manager import * from pycassa.util import * from pycassa.types import * +from sandesh_common.vns.constants import DISCOVERY_SERVER_KEYSPACE_NAME, \ + CASSANDRA_DEFAULT_GC_GRACE_SECONDS class DiscoveryCassandraClient(object): - _DISCOVERY_KEYSPACE_NAME = 'DISCOVERY_SERVER' + _DISCOVERY_KEYSPACE_NAME = DISCOVERY_SERVER_KEYSPACE_NAME _DISCOVERY_CF_NAME = 'discovery' @classmethod @@ -99,7 +101,8 @@ def _cassandra_ensure_keyspace(self, server_list, (cf_name, comparator_type, validator_type) = cf_info sys_mgr.create_column_family(keyspace_name, cf_name, comparator_type = comparator_type, default_validation_class = validator_type) - sys_mgr.alter_column_family(keyspace_name, cf_name, gc_grace_seconds=0) + sys_mgr.alter_column_family(keyspace_name, cf_name, + gc_grace_seconds=CASSANDRA_DEFAULT_GC_GRACE_SECONDS) except pycassa.cassandra.ttypes.InvalidRequestException as e: # TODO verify only EEXISTS print "Warning! " + str(e) diff --git a/src/discovery/disc_server.py b/src/discovery/disc_server.py index 0c32c0c1175..96385bab72f 100644 --- a/src/discovery/disc_server.py +++ b/src/discovery/disc_server.py @@ -8,10 +8,10 @@ """ import gevent -from disc_cassdb import DiscoveryCassandraClient from gevent import monkey monkey.patch_all() from gevent import hub +from disc_cassdb import DiscoveryCassandraClient import sys import time diff --git a/src/nodemgr/database_nodemgr/database_event_manager.py b/src/nodemgr/database_nodemgr/database_event_manager.py index 0d90678206c..1b8b26312bc 100644 --- a/src/nodemgr/database_nodemgr/database_event_manager.py +++ b/src/nodemgr/database_nodemgr/database_event_manager.py @@ -27,7 +27,8 @@ from pysandesh.gen_py.sandesh_trace.ttypes import SandeshTraceRequest from sandesh_common.vns.ttypes import Module, NodeType from sandesh_common.vns.constants import ModuleNames, NodeTypeNames,\ - Module2NodeType, INSTANCE_ID_DEFAULT, SERVICE_CONTRAIL_DATABASE + Module2NodeType, INSTANCE_ID_DEFAULT, SERVICE_CONTRAIL_DATABASE, \ + RepairNeededKeyspaces from subprocess import Popen, PIPE from StringIO import StringIO @@ -43,7 +44,7 @@ class DatabaseEventManager(EventManager): def __init__(self, rule_file, discovery_server, discovery_port, collector_addr, - hostip, minimum_diskgb): + hostip, minimum_diskgb, cassandra_repair_interval): EventManager.__init__( self, rule_file, discovery_server, discovery_port, collector_addr) @@ -52,6 +53,7 @@ def __init__(self, rule_file, discovery_server, self.module_id = ModuleNames[self.module] self.hostip = hostip self.minimum_diskgb = minimum_diskgb + self.cassandra_repair_interval = cassandra_repair_interval self.supervisor_serverurl = "unix:///tmp/supervisord_database.sock" self.add_current_process() # end __init__ @@ -193,6 +195,14 @@ def database_periodic(self): # end database_periodic + def cassandra_repair(self): + for keyspace in RepairNeededKeyspaces: + repair_file_name = '/var/log/cassandra/repair-' + keyspace + '.log' + with open(repair_file_name, "a") as repair_file: + subprocess.Popen(["nodetool", "repair", "-pr", keyspace], + stdout=repair_file, stderr=repair_file) + #end cassandra_repair + def send_disk_usage_info(self): self.send_disk_usage_info_base( NodeStatusUVE, NodeStatus, DiskPartitionUsageStats) @@ -222,4 +232,7 @@ def runforever(self, test=False): if headers['eventname'].startswith("TICK_60"): self.database_periodic() prev_current_time = self.event_tick_60(prev_current_time) + # Perform nodetool repair every cassandra_repair_interval hours + if self.tick_count % (60 * self.cassandra_repair_interval) == 0: + self.cassandra_repair() self.listener_nodemgr.ok(self.stdout) diff --git a/src/nodemgr/main.py b/src/nodemgr/main.py index 109a342158b..aad6ea4aa8c 100755 --- a/src/nodemgr/main.py +++ b/src/nodemgr/main.py @@ -67,7 +67,8 @@ def main(args_str=' '.join(sys.argv[1:])): default = {'rules': '', 'collectors': [], 'hostip': '127.0.0.1', - 'minimum_diskgb': 256 + 'minimum_diskgb': 256, + 'cassandra_repair_interval': 24, } node_type = args.nodetype if (node_type == 'contrail-analytics'): @@ -119,6 +120,9 @@ def main(args_str=' '.join(sys.argv[1:])): help="Minimum disk space in GB's") parser.add_argument("--hostip", help="IP address of host") + parser.add_argument("--cassandra_repair_interval", type=int, + help="Time in hours to periodically run " + "nodetool repair for cassandra maintenance") try: _args = parser.parse_args(remaining_argv) except: @@ -157,10 +161,11 @@ def main(args_str=' '.join(sys.argv[1:])): elif (node_type == 'contrail-database'): hostip = _args.hostip minimum_diskgb = _args.minimum_diskgb + cassandra_repair_interval = _args.cassandra_repair_interval prog = DatabaseEventManager( rule_file, discovery_server, discovery_port, collector_addr, - hostip, minimum_diskgb) + hostip, minimum_diskgb, cassandra_repair_interval) else: sys.stderr.write("Node type" + str(node_type) + "is incorrect" + "\n") return diff --git a/src/sandesh/common/vns.sandesh b/src/sandesh/common/vns.sandesh index 3904d22ef9a..7bd745c8b72 100644 --- a/src/sandesh/common/vns.sandesh +++ b/src/sandesh/common/vns.sandesh @@ -432,3 +432,19 @@ const string ALARM_GENERATOR_SERVICE_NAME = "AlarmGenerator" const string IFMAP_SERVER_DISCOVERY_SERVICE_NAME = "IfmapServer" const string XMPP_SERVER_DISCOVERY_SERVICE_NAME = "xmpp-server" const string DNS_SERVER_DISCOVERY_SERVICE_NAME = "dns-server" + +// Cassandra Keyspace Names +const string SVC_MONITOR_KEYSPACE_NAME = "svc_monitor_keyspace" +const string SCHEMA_KEYSPACE_NAME = "to_bgp_keyspace" +const string API_SERVER_KEYSPACE_NAME = "config_db_uuid" +const string DISCOVERY_SERVER_KEYSPACE_NAME = "DISCOVERY_SERVER" + +const list RepairNeededKeyspaces = [ + API_SERVER_KEYSPACE_NAME, + SCHEMA_KEYSPACE_NAME, + SVC_MONITOR_KEYSPACE_NAME, + DISCOVERY_SERVER_KEYSPACE_NAME, +] + +// Set to default of 10 days +const u32 CASSANDRA_DEFAULT_GC_GRACE_SECONDS = 864000