From c1563d8c9217943b0bed7e97d22d139e79040270 Mon Sep 17 00:00:00 2001 From: Sanju Abraham Date: Thu, 12 Feb 2015 19:39:51 -0800 Subject: [PATCH] Close-Bug:1408756. This bug address the keystone token and cmon purge issue during scale and failure conditions Change-Id: I9429bd75315b165f8b28592723de60da92a137ea --- .../openstack/ha/galera_setup.py | 2 +- .../scripts/contrail-cmon-monitor.sh | 2 + .../openstack/scripts/contrail-token-clean.sh | 53 +++++++++++++------ 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/contrail_provisioning/openstack/ha/galera_setup.py b/contrail_provisioning/openstack/ha/galera_setup.py index 73492a5c..9343edd6 100755 --- a/contrail_provisioning/openstack/ha/galera_setup.py +++ b/contrail_provisioning/openstack/ha/galera_setup.py @@ -213,7 +213,7 @@ def setup_grants(self): def setup_cron(self): with settings(hide('everything'), warn_only=True): local('crontab -l > %s/galera_cron' % self._temp_dir_name) - local('echo "0 0 * * * /opt/contrail/bin/contrail-token-clean.sh" >> %s/galera_cron' % self._temp_dir_name) + local('echo "0,30 * * * * /opt/contrail/bin/contrail-token-clean.sh" >> %s/galera_cron' % self._temp_dir_name) local('crontab %s/galera_cron' % self._temp_dir_name) local('rm %s/galera_cron' % self._temp_dir_name) diff --git a/contrail_provisioning/openstack/scripts/contrail-cmon-monitor.sh b/contrail_provisioning/openstack/scripts/contrail-cmon-monitor.sh index 937f967b..163e60e4 100755 --- a/contrail_provisioning/openstack/scripts/contrail-cmon-monitor.sh +++ b/contrail_provisioning/openstack/scripts/contrail-cmon-monitor.sh @@ -43,6 +43,7 @@ SET_CMON_STATS_COLL_PARAM="update cmon_configuration set value=1440 where param= SET_CMON_HOST_COLL_PARAM="update cmon_configuration set value=1440 where param='host_stats_collection_interval';" SET_CMON_LOG_COLL_PARAM="update cmon_configuration set value=1440 where param='log_collection_interval';" SET_CMON_STATS_PARAM="update cmon_configuration set value=720 where param='db_hourly_stats_collection_interval';" +SET_CMON_BACKUP_RETENTION="update cmon_configuration set value=1 where param='BACKUP_RETENTION';" timestamp() { date +"%T" @@ -166,6 +167,7 @@ if [ $viponme -eq 1 ]; then mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_HOST_COLL_PARAM}" mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_LOG_COLL_PARAM}" mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_STATS_PARAM}" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_BACKUP_RETENTION}" log_info_msg "Done setting params for cmon" (exec $RMQ_MONITOR)& diff --git a/contrail_provisioning/openstack/scripts/contrail-token-clean.sh b/contrail_provisioning/openstack/scripts/contrail-token-clean.sh index 270d91e1..10e8767f 100755 --- a/contrail_provisioning/openstack/scripts/contrail-token-clean.sh +++ b/contrail_provisioning/openstack/scripts/contrail-token-clean.sh @@ -4,13 +4,17 @@ # validated the request. # Author - Sanju Abraham +source /etc/contrail/ha/cmon_param + LOGFILE=/var/log/contrail/ha/token-cleanup.log mysql_user=keystone mysql_password=keystone -mysql_host=localhost +mysql_host=$VIP +mysql_port=33306 mysql=$(which mysql) cmon_user_pass=cmon cmon_stats_purge="call sp_cmon_purge_history;" +viponme=0 timestamp() { date +"%T" @@ -31,24 +35,41 @@ log_info_msg() { echo "$(timestamp): INFO: $msg" >> $LOGFILE } -log_info_msg "keystone-cleaner::Starting Keystone 'token' table cleanup" - log_info_msg "keystone-cleaner::Starting token cleanup" -mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -e 'USE keystone ; DELETE FROM token WHERE NOT DATE_SUB(CURDATE(),INTERVAL 1 DAY) <= expires;' -valid_token=$($mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -e 'USE keystone ; SELECT * FROM token;' | wc -l) -log_info_msg "keystone-cleaner::Finishing token cleanup, there is still $valid_token valid tokens..." + + mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -P${mysql_port} -e "USE keystone ; DELETE FROM token where expires <= convert_tz(now(),@@session.time_zone,'+01:00');" + valid_token=$($mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -P${mysql_port} -e "USE keystone ; SELECT count(*) FROM token;") + valid_token=$(echo $valid_token | awk '{print $2}') + +log_info_msg "keystone-cleaner::Finishing token cleanup, there are $valid_token valid tokens..." + + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; ${cmon_stats_purge}" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_log;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table memory_usage_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cpu_stats_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table ram_stats_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table disk_stats_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table diskdata_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_global_statistics_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_statistics_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_advisor_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table expression_result_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_performance_results;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table net_stats_history;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_job;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_job_message;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cluster_log;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table backup_log;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table restore_log;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table backup;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table restore;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table alarm_log;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_query_histogram;" + mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_slow_queries;" + +log_info_msg "Purged cmon stats history" find /var/log/contrail/ha/ -size +10240k -exec rm -f {} \; find /var/log/cmon.log -size +10240k -exec rm -f {} \; -# Restricting purge cmon logs to be run in only one server. -# Run the purge script only if cmon is running in the server. -killall -q -0 cmon -if [ $? -eq "0" ]; then - mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -e "use cmon; ${cmon_stats_purge}" - log_info_msg "Purged cmon stats history" -else - log_info_msg "CMON is not running, hence skipping to purge cmon stats history" -fi - exit 0