Close-Bug:1408756. This bug address the keystone token and cmon purge…

… issue during scale and failure conditions Change-Id: I9429bd75315b165f8b28592723de60da92a137ea
Juniper · Feb 13, 2015 · c1563d8 · c1563d8
1 parent 4535773
commit c1563d8
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 17 deletions.
diff --git a/contrail_provisioning/openstack/ha/galera_setup.py b/contrail_provisioning/openstack/ha/galera_setup.py
@@ -213,7 +213,7 @@ def setup_grants(self):
     def setup_cron(self):
         with settings(hide('everything'), warn_only=True):
             local('crontab -l > %s/galera_cron' % self._temp_dir_name)
-        local('echo "0 0 * * * /opt/contrail/bin/contrail-token-clean.sh" >> %s/galera_cron' % self._temp_dir_name)
+        local('echo "0,30 * * * * /opt/contrail/bin/contrail-token-clean.sh" >> %s/galera_cron' % self._temp_dir_name)
         local('crontab %s/galera_cron' % self._temp_dir_name)
         local('rm %s/galera_cron' % self._temp_dir_name)
 

diff --git a/contrail_provisioning/openstack/scripts/contrail-cmon-monitor.sh b/contrail_provisioning/openstack/scripts/contrail-cmon-monitor.sh
@@ -43,6 +43,7 @@ SET_CMON_STATS_COLL_PARAM="update cmon_configuration set value=1440 where param=
 SET_CMON_HOST_COLL_PARAM="update cmon_configuration set value=1440 where param='host_stats_collection_interval';"
 SET_CMON_LOG_COLL_PARAM="update cmon_configuration set value=1440 where param='log_collection_interval';"
 SET_CMON_STATS_PARAM="update cmon_configuration set value=720 where param='db_hourly_stats_collection_interval';"
+SET_CMON_BACKUP_RETENTION="update cmon_configuration set value=1 where param='BACKUP_RETENTION';"
 
 timestamp() {
     date +"%T"
@@ -166,6 +167,7 @@ if [ $viponme -eq 1 ]; then
       mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_HOST_COLL_PARAM}"
       mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_LOG_COLL_PARAM}"
       mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_STATS_PARAM}"
+      mysql -u${cmon_user_pass} -p${cmon_user_pass} -e "USE cmon; ${SET_CMON_BACKUP_RETENTION}"
       log_info_msg "Done setting params for cmon"
 
       (exec $RMQ_MONITOR)&

diff --git a/contrail_provisioning/openstack/scripts/contrail-token-clean.sh b/contrail_provisioning/openstack/scripts/contrail-token-clean.sh
@@ -4,13 +4,17 @@
 # validated the request.
 # Author - Sanju Abraham
 
+source /etc/contrail/ha/cmon_param
+
 LOGFILE=/var/log/contrail/ha/token-cleanup.log
 mysql_user=keystone
 mysql_password=keystone
-mysql_host=localhost
+mysql_host=$VIP
+mysql_port=33306
 mysql=$(which mysql)
 cmon_user_pass=cmon
 cmon_stats_purge="call sp_cmon_purge_history;"
+viponme=0
 
 timestamp() {
     date +"%T"
@@ -31,24 +35,41 @@ log_info_msg() {
     echo "$(timestamp): INFO: $msg" >> $LOGFILE
 }
 
-log_info_msg "keystone-cleaner::Starting Keystone 'token' table cleanup"
-
 log_info_msg "keystone-cleaner::Starting token cleanup"
-mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -e 'USE keystone ; DELETE FROM token WHERE NOT DATE_SUB(CURDATE(),INTERVAL 1 DAY) <= expires;'
-valid_token=$($mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -e 'USE keystone ; SELECT * FROM token;' | wc -l)
-log_info_msg "keystone-cleaner::Finishing token cleanup, there is still $valid_token valid tokens..."
+
+    mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -P${mysql_port} -e "USE keystone ; DELETE FROM token where expires <= convert_tz(now(),@@session.time_zone,'+01:00');"
+    valid_token=$($mysql -u${mysql_user} -p${mysql_password} -h${mysql_host} -P${mysql_port} -e "USE keystone ; SELECT count(*) FROM token;")
+    valid_token=$(echo $valid_token | awk '{print $2}')
+
+log_info_msg "keystone-cleaner::Finishing token cleanup, there are $valid_token valid tokens..."
+
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; ${cmon_stats_purge}"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_log;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table memory_usage_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cpu_stats_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table ram_stats_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table disk_stats_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table diskdata_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_global_statistics_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_statistics_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_advisor_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table expression_result_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_performance_results;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table net_stats_history;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_job;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cmon_job_message;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table cluster_log;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table backup_log;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table restore_log;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table backup;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table restore;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table alarm_log;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_query_histogram;"
+    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -P${mysql_port} -e "use cmon; truncate table mysql_slow_queries;"
+
+log_info_msg "Purged cmon stats history"
 
 find /var/log/contrail/ha/ -size +10240k -exec rm -f {} \;
 find /var/log/cmon.log -size +10240k -exec rm -f {} \;
 
-# Restricting purge cmon logs to be run in only one server.
-# Run the purge script only if cmon is running in the server.
-killall -q -0 cmon
-if [ $? -eq "0" ]; then
-    mysql -u${cmon_user_pass} -p${cmon_user_pass} -h${mysql_host} -e "use cmon; ${cmon_stats_purge}"
-    log_info_msg "Purged cmon stats history"
-else
-    log_info_msg "CMON is not running, hence skipping to purge cmon stats history"
-fi
-
 exit 0