Skip to content

Commit

Permalink
Merge "Related-Bug: #1581650 Add cache for table_type STAT queries, a…
Browse files Browse the repository at this point in the history 
…nd upon forcerefresh do backend query."
  • Loading branch information
@opencontrail-ci-admin
Zuul authored and opencontrail-ci-admin committed Aug 5, 2016
2 parents 78d1e68 + d98e4d8 commit b38d278
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 10 deletions.
7 changes: 7 additions & 0 deletions webroot/reports/qe/ui/api/parseURL.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@
<timeout>600</timeout>
<feature>query-engine</feature>
<callback>qeapi.runPOSTQuery</callback>
</item>
<item>
<url>/api/qe/get-query</url>
<method>post</method>
<timeout>600</timeout>
<feature>query-engine</feature>
<callback>qeapi.getQueryData</callback>
</item>
<item>
<url>/api/qe/query/export</url>
Expand Down
103 changes: 93 additions & 10 deletions webroot/reports/qe/ui/api/query.api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ var qeapi = module.exports,
redisReadStream = require('redis-rstream'),
Worker = require('webworker-threads').Worker;
qs = require('querystring'),
crypto = require('crypto'),
redisUtils = require(process.mainModule.exports["corePath"] +
'/src/serverroot/utils/redis.utils'),
_ = require('underscore');
Expand Down Expand Up @@ -180,7 +181,7 @@ function getCurrentTime(req, res) {
commonUtils.handleJSONResponse(null, res, {currentTime: currentTime});
};

function runQuery(req, res, queryReqObj, appData) {
function runQuery(req, res, queryReqObj, appData, isGetQ) {
var queryId = queryReqObj['queryId'],
chunk = queryReqObj['chunk'], chunkSize = parseInt(queryReqObj['chunkSize']),
sort = queryReqObj['sort'], cachedResultConfig;
Expand All @@ -197,20 +198,20 @@ function runQuery(req, res, queryReqObj, appData) {
} else if (exists == 1) {
returnCachedQueryResult(res, cachedResultConfig, handleQueryResponse);
} else {
runNewQuery(req, res, queryId, queryReqObj, appData);
runNewQuery(req, res, queryId, queryReqObj, appData, isGetQ);
}
});
} else {
runNewQuery(req, res, null, queryReqObj, appData);
runNewQuery(req, res, null, queryReqObj, appData, isGetQ);
}
};

function runNewQuery(req, res, queryId, queryReqObj, appData) {
function runNewQuery(req, res, queryId, queryReqObj, appData, isGetQ) {
var queryOptions = getQueryOptions(queryReqObj),
queryJSON = getQueryJSON4Table(queryReqObj);

queryOptions.queryJSON = queryJSON;
executeQuery(res, queryOptions, appData);
executeQuery(res, queryOptions, appData, isGetQ);
};

function getQueryOptions(queryReqObj) {
Expand All @@ -234,7 +235,7 @@ function getQueryOptions(queryReqObj) {
return queryOptions;
};

function executeQuery(res, queryOptions, appData) {
function executeQuery(res, queryOptions, appData, isGetQ) {
var queryJSON = queryOptions.queryJSON,
async = queryOptions.async, asyncHeader = {"Expect": "202-accepted"};

Expand All @@ -252,7 +253,7 @@ function executeQuery(res, queryOptions, appData) {
queryOptions['intervalId'] = setInterval(fetchQueryResults, queryOptions.pollingInterval, res, jsonData, queryOptions, appData);
queryOptions['timeoutId'] = setTimeout(stopFetchQueryResult, queryOptions.pollingTimeout, queryOptions);
} else {
processQueryResults(res, jsonData, queryOptions);
processQueryResults(res, jsonData, queryOptions, isGetQ);
}
}, async ? asyncHeader : {});
};
Expand Down Expand Up @@ -524,7 +525,82 @@ function updateQueryStatus(queryOptions) {
redisClient.hmset(queryOptions.queryQueue, queryOptions.queryId, JSON.stringify(queryStatus));
};

function processQueryResults(res, queryResults, queryOptions) {
function createStatRedisKey (req, query)
{
var urlReq = require('url');
var urlParts = urlReq.parse(req.url);
var reqPayload = commonUtils.cloneObj(query);
var fromTime =
commonUtils.getValueByJsonPath(reqPayload, 'formModelAttrs;from_time',
null);
if (null != fromTime) {
delete reqPayload.formModelAttrs.from_time;
}
var fromTimeUTC =
commonUtils.getValueByJsonPath(reqPayload,
'formModelAttrs;from_time_utc', null);
if (null != fromTimeUTC) {
delete reqPayload.formModelAttrs.from_time_utc;
}
var toTime =
commonUtils.getValueByJsonPath(reqPayload,
'formModelAttrs;to_time', null);
if (null != toTime) {
delete reqPayload.formModelAttrs.to_time;
}
var toTimeUTC =
commonUtils.getValueByJsonPath(reqPayload,
'formModelAttrs;to_time_utc', null);
if (null != toTimeUTC) {
delete reqPayload.formModelAttrs.to_time_utc;
}
var reqPayload = commonUtils.doDeepSort(reqPayload);
var md5Data = urlParts.pathname + JSON.stringify(reqPayload);
var redisKey =
crypto.createHash('md5').update(md5Data).digest('hex');
return redisKey;
}

function saveDataToRedisByReqPayload (res, resJson)
{
var reqPayload = res.req.body;
if (global.HTTP_REQUEST_GET == res.req.method) {
reqPayload = res.req.query;
} else {
reqPayload = res.req.body;
}
var redisKey = createStatRedisKey(res.req, reqPayload);
redisClient.set(redisKey, JSON.stringify(resJson), function(error) {
if (null != error) {
logutils.logger.error('Redis key ' + redisKey + ' save error:' +
error);
}
});
}

function getQueryData (req, res, appData)
{
var query;
if (global.HTTP_REQUEST_GET == req.method) {
query = req.query;
} else {
query = req.body;
}
if ((null != req.query) && ('forceRefresh' in req.query)) {
runQuery(req, res, query, appData, true);
return;
}
var redisKey = createStatRedisKey(req, query);
redisClient.get(redisKey, function(error, data) {
if ((null != error) || (null == data)) {
runQuery(req, res, query, appData, true);
return;
}
commonUtils.handleJSONResponse(null, res, JSON.parse(data));
});
}

function processQueryResults(res, queryResults, queryOptions, isGetQ) {
var startDate = new Date(), startTime = startDate.getTime(),
queryId = queryOptions.queryId, chunkSize = queryOptions.chunkSize,
queryJSON = queryOptions.queryJSON, endDate = new Date(),
Expand All @@ -543,7 +619,12 @@ function processQueryResults(res, queryResults, queryOptions) {
} else {
responseJSON = resultJSON.slice(0, chunkSize);
}
commonUtils.handleJSONResponse(null, res, {data: responseJSON, total: total, queryJSON: queryJSON, chunk: 1, chunkSize: chunkSize, serverSideChunking: true});
var resJson = {data: responseJSON, total: total, queryJSON: queryJSON,
chunk: 1, chunkSize: chunkSize, serverSideChunking: true};
commonUtils.handleJSONResponse(null, res, resJson);
if (true == isGetQ) {
saveDataToRedisByReqPayload(res, resJson);
}
}

if(queryId != null) {
Expand Down Expand Up @@ -1158,4 +1239,6 @@ exports.deleteQueryCache4Queue = deleteQueryCache4Queue;
exports.flushQueryCache = flushQueryCache;
exports.exportQueryResult = exportQueryResult;
exports.getQueryJSON4Table = getQueryJSON4Table;
exports.getCurrentTime = getCurrentTime;
exports.getCurrentTime = getCurrentTime;
exports.getQueryData = getQueryData;

0 comments on commit b38d278

Please sign in to comment.