Skip to content

Commit

Permalink
Fixes: This bug address the keystone token and cmon purge issue durin…
Browse files Browse the repository at this point in the history 
…g scale and failure conditions

Galera cluster was having issues when DELETE calls where made to large tables like CMON.

This fix slows down the log collection and runs the command in only one node.

Change-Id: I77afc8854b8c6c284daa1b812666a57328dc7eca
Close-Bug:1408756
  • Loading branch information
Ranjeet R committed Feb 16, 2015
1 parent 5313dbf commit 94be5bd
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 7 deletions.
2 changes: 1 addition & 1 deletion contrail_provisioning/openstack/ha/galera_setup.py
View file
Expand Up @@ -213,7 +213,7 @@ def setup_grants(self):
def setup_cron(self):
with settings(hide('everything'), warn_only=True):
local('crontab -l > %s/galera_cron' % self._temp_dir_name)
local('echo "0 0 * * * /opt/contrail/bin/contrail-token-clean.sh" >> %s/galera_cron' % self._temp_dir_name)
local('echo "0,30 * * * * /opt/contrail/bin/contrail-token-clean.sh" >> %s/galera_cron' % self._temp_dir_name)
local('crontab %s/galera_cron' % self._temp_dir_name)
local('rm %s/galera_cron' % self._temp_dir_name)

Expand Down
2 changes: 2 additions & 0 deletions contrail_provisioning/openstack/scripts/contrail-cmon-monitor.sh
View file
Expand Up @@ -43,6 +43,7 @@ SET_CMON_STATS_COLL_PARAM="update cmon_configuration set value=1440 where param=
SET_CMON_HOST_COLL_PARAM="update cmon_configuration set value=1440 where param='host_stats_collection_interval';"
SET_CMON_LOG_COLL_PARAM="update cmon_configuration set value=1440 where param='log_collection_interval';"
SET_CMON_STATS_PARAM="update cmon_configuration set value=720 where param='db_hourly_stats_collection_interval';"
SET_CMON_BACKUP_RETENTION="update cmon_configuration set value=1 where param='BACKUP_RETENTION';"

timestamp() {
date +"%T"
Expand Down Expand Up @@ -166,6 +167,7 @@ if [ $viponme -eq 1 ]; then
mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_HOST_COLL_PARAM}"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_LOG_COLL_PARAM}"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_STATS_PARAM}"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_BACKUP_RETENTION}"
log_info_msg "Done setting params for cmon"

(exec $RMQ_MONITOR)&
Expand Down
45 changes: 39 additions & 6 deletions contrail_provisioning/openstack/scripts/contrail-token-clean.sh
View file
Expand Up @@ -4,10 +4,13 @@
# validated the request.
# Author - Sanju Abraham

source /etc/contrail/ha/cmon_param

LOGFILE=/var/log/contrail/ha/token-cleanup.log
mysql_user=keystone
mysql_password=keystone
mysql_host=localhost
mysql_host=$VIP
mysql_port=33306
mysql=$(which mysql)
cmon_user_pass=cmon
cmon_stats_purge="call sp_cmon_purge_history;"
Expand All @@ -31,16 +34,46 @@ log_info_msg() {
echo "$(timestamp): INFO: $msg" >> $LOGFILE
}

log_info_msg "keystone-cleaner::Starting Keystone 'token' table cleanup"

log_info_msg "keystone-cleaner::Starting token cleanup"
mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -e 'USE keystone ; DELETE FROM token WHERE NOT DATE_SUB(CURDATE(),INTERVAL 1 DAY) <= expires;'
valid_token=$($mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -e 'USE keystone ; SELECT * FROM token;' | wc -l)

mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -P${mysql_port} -e "USE keystone ; DELETE FROM token where expires <= convert_tz(now(),@@session.time_zone,'+01:00');"
valid_token=$($mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -P${mysql_port} -e "USE keystone ; SELECT count(*) FROM token;")
valid_token=$(echo $valid_token | awk '{print $2}')
log_info_msg "keystone-cleaner::Finishing token cleanup, there is still $valid_token valid tokens..."

killall -q -0 cmon
if [ $? -eq "0" ]; then
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; ${cmon_stats_purge}"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_log;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table memory_usage_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cpu_stats_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table ram_stats_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table disk_stats_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table diskdata_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_global_statistics_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_statistics_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_advisor_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table expression_result_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_performance_results;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table net_stats_history;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_job;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_job_message;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cluster_log;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table backup_log;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table restore_log;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table backup;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table restore;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table alarm_log;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_query_histogram;"
mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_slow_queries;"

log_info_msg "Purged cmon stats history"
else
log_info_msg "CMON is not running, hence skipping to purge cmon stats history"
fi

find /var/log/contrail/ha/ -size +10240k -exec rm -f {} \;
find /var/log/cmon.log -size +10240k -exec rm -f {} \;

mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -e "use cmon; ${cmon_stats_purge}"
log_info_msg "Purged cmon stats history"
exit 0

0 comments on commit 94be5bd

Please sign in to comment.