diff --git a/src/analytics/OpServerProxy.cc b/src/analytics/OpServerProxy.cc index bb967c50022..1ebfe0ee7b8 100644 --- a/src/analytics/OpServerProxy.cc +++ b/src/analytics/OpServerProxy.cc @@ -351,6 +351,10 @@ class OpServerProxy::OpServerImpl { LOG(DEBUG, "NULL Reply...\n"); return; } + // If redis returns error for async request, then perhaps it + // is busy executing a script and it has reached the maximum + // execution time limit. + assert(reply->type != REDIS_REPLY_ERROR); if (rpi) { rpi->ProcessCallback(reply); diff --git a/src/analytics/delrequest.lua b/src/analytics/delrequest.lua index d7f40fa003c..3dd78ca7e24 100644 --- a/src/analytics/delrequest.lua +++ b/src/analytics/delrequest.lua @@ -27,10 +27,17 @@ redis.log(redis.LOG_NOTICE,"DelRequest for "..sm) local db = tonumber(ARGV[5]) redis.call('select',db) local typ = redis.call('smembers',"TYPES:"..sm) +--- In a scaled setup, sub_del() can be bottleneck. +--- Therefore, call sub_del() only for types, where aggtype="stats" is specified +--- aggtype="stats" is deprecated and will not be supported from next release +local stat_types = {ModuleServerState=true, ModuleCpuState=true, + BgpRouterState=true, VrouterStatsAgent=true} for k,v in pairs(typ) do + redis.log(redis.LOG_NOTICE, "Read UVES:"..sm..":"..v) local lres = redis.call('zrange',"UVES:"..sm..":"..v, 0, -1, "withscores") local iter = 1 + redis.log(redis.LOG_NOTICE, "Delete "..sm..":"..v.." [#"..(#lres/2).."]") while iter <= #lres do local deltyp = v local deluve = lres[iter] @@ -47,10 +54,11 @@ for k,v in pairs(typ) do redis.call('hdel', "KEY2PART:"..sm..":"..deltyp, deluve) redis.call('srem', "PART2KEY:"..part, sm..":"..deltyp..":"..deluve) end - redis.log(redis.LOG_NOTICE,"DEL for "..dkey.." part "..part) local dval = "VALUES:"..deluve..":"..sm..":"..deltyp - sub_del(dval) + if stat_types[deltyp] then + sub_del(dval) + end local lttt = redis.call('exists', dval) if lttt == 1 then diff --git a/src/analytics/redis_processor_vizd.cc b/src/analytics/redis_processor_vizd.cc index 15c6d74a396..3796e6c1c2d 100644 --- a/src/analytics/redis_processor_vizd.cc +++ b/src/analytics/redis_processor_vizd.cc @@ -203,10 +203,11 @@ RedisProcessorExec::SyncDeleteUVEs(const std::string & redis_ip, unsigned short const std::string &module, const std::string &instance_id) { redisContext *c = redisConnect(redis_ip.c_str(), redis_port); + std::string generator(source + ":" + node_type + ":" + module + + ":" + instance_id); if (c->err) { - LOG(ERROR, "No connection for SyncDeleteUVEs " << source << ":" << - node_type << ":" << module << ":" << instance_id); + LOG(ERROR, "No connection for SyncDeleteUVEs : " << generator); redisFree(c); return false; } @@ -233,7 +234,8 @@ RedisProcessorExec::SyncDeleteUVEs(const std::string & redis_ip, unsigned short module.c_str(), instance_id.c_str(), integerToString(REDIS_DB_UVE).c_str()); if (!reply) { - LOG(INFO, "SyncDeleteUVEs Error : " << c->errstr); + LOG(ERROR, "SyncDeleteUVEs failed for " << generator << " : " << + c->errstr); redisFree(c); return false; } @@ -243,11 +245,14 @@ RedisProcessorExec::SyncDeleteUVEs(const std::string & redis_ip, unsigned short redisFree(c); return true; } - LOG(ERROR, "Unrecognized reponse of type " << reply->type << - " for SyncDeleteUVEs " << source << ":" << node_type << ":" << - module << ":" << instance_id); + LOG(ERROR, "Unrecognized response of type " << reply->type << + " for SyncDeleteUVEs : " << generator); freeReplyObject(reply); - redisFree(c); + redisFree(c); + // Redis returns error if the time taken to execute the script is + // more than the lua-time-limit configured in redis.conf + // It is not easy to handle this case gracefully, hence assert. + assert(0); return false; }