Merge "Discovery Client needs to be more resilient to loads on Discov…

…ery Server." into R2.21.x
Juniper · Feb 3, 2016 · 2f60ab1 · 2f60ab1
2 parents 681342a + 8ea2799
commit 2f60ab1
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 7 deletions.
diff --git a/src/discovery/client/discovery_client.cc b/src/discovery/client/discovery_client.cc
@@ -77,7 +77,7 @@ DSPublishResponse::DSPublishResponse(std::string serviceName,
                           TaskScheduler::GetInstance()->GetTaskId("http client"), 0)),
       ds_client_(ds_client), publish_msg_(""), attempts_(0),
       pub_sent_(0), pub_rcvd_(0), pub_fail_(0), pub_fallback_(0),
-      pub_hb_sent_(0), pub_hb_fail_(0), pub_hb_rcvd_(0),
+      pub_hb_sent_(0), pub_hb_fail_(0), pub_hb_rcvd_(0), pub_hb_timeout_(0),
       publish_cb_called_(false), heartbeat_cb_called_(false) {
 }
 
@@ -278,7 +278,9 @@ void DiscoveryServiceClient::PublishResponseHandler(std::string &xmls,
         // Errorcode is of type CURLcode
         if (ec.value() != 0) {
             DISCOVERY_CLIENT_TRACE(DiscoveryClientErrorMsg,
-                "PublishResponseHandler Error", serviceName, ec.value());
+                "Error PublishResponseHandler ",
+                 serviceName + " " + curl_error_category.message(ec.value()),
+                 ec.value());
             // exponential back-off and retry
             resp->pub_fail_++;
             resp->attempts_++; 
@@ -723,8 +725,9 @@ void DiscoveryServiceClient::SubscribeResponseHandler(std::string &xmls,
         // Errorcode is of type CURLcode
         if (ec.value() != 0) {
             DISCOVERY_CLIENT_TRACE(DiscoveryClientErrorMsg,
-                "SubscribeResponseHandler Error",
-                 serviceName, ec.value());
+                "Error SubscribeResponseHandler ",
+                 serviceName + " " + curl_error_category.message(ec.value()),
+                 ec.value());
             // exponential back-off and retry
             hdr->attempts_++; 
             hdr->sub_fail_++;
@@ -928,16 +931,34 @@ void DiscoveryServiceClient::HeartBeatResponseHandler(std::string &xmls,
         // Errorcode is of type CURLcode
         if (ec.value() != 0) {
             DISCOVERY_CLIENT_TRACE(DiscoveryClientErrorMsg,
-                "HeartBeatResponseHandler Error", serviceName, ec.value());
+                "Error HeartBeatResponseHandler ",
+                 serviceName + " " + curl_error_category.message(ec.value()),
+                 ec.value());
+
             resp->pub_hb_fail_++;
-            resp->StopHeartBeatTimer();
             // Resend original publish request after exponential back-off
             resp->attempts_++;
+            // CURLE_OPERATION_TIMEDOUT, timeout triggered when no
+            // response from Discovery Server for 4secs.
+            if (curl_error_category.message(ec.value()).find("Timeout") !=
+                std::string::npos) {
+                resp->pub_hb_timeout_++;
+                if (resp->attempts_ <= 3) {
+                    // Make client resilient to heart beat misses.
+                    // Continue sending heartbeat
+                    return;
+                } else {
+                    // reset attempts so publish can be sent right away
+                    resp->attempts_ = 0;
+                }
+            }
+
             // Update connection info
             ConnectionState::GetInstance()->Update(ConnectionType::DISCOVERY,
                 serviceName, ConnectionStatus::DOWN, ds_endpoint_,
                 ec.message());
-             resp->StartPublishConnectTimer(resp->GetConnectTime());
+            resp->StopHeartBeatTimer();
+            resp->StartPublishConnectTimer(resp->GetConnectTime());
         } else if (resp->heartbeat_cb_called_ == false) {
             DISCOVERY_CLIENT_TRACE(DiscoveryClientErrorMsg,
                 "HeartBeatResponseHandler, Only header received",
@@ -973,6 +994,7 @@ void DiscoveryServiceClient::HeartBeatResponseHandler(std::string &xmls,
             return;
     }
     resp->pub_hb_rcvd_++;
+    resp->attempts_ = 0;
 }
 
 void DiscoveryServiceClient::SendHttpPostMessage(std::string msg_type, 
@@ -1074,6 +1096,7 @@ void DiscoveryServiceClient::FillDiscoveryServicePublisherStats(
          stats.set_heartbeat_sent(pub_resp->pub_hb_sent_);
          stats.set_heartbeat_fail(pub_resp->pub_hb_fail_);   
          stats.set_heartbeat_rcvd(pub_resp->pub_hb_rcvd_);
+         stats.set_heartbeat_timeout(pub_resp->pub_hb_timeout_);
          stats.set_publish_sent(pub_resp->pub_sent_);
          stats.set_publish_rcvd(pub_resp->pub_rcvd_); 
          stats.set_publish_fail(pub_resp->pub_fail_); 

diff --git a/src/discovery/client/discovery_client.h b/src/discovery/client/discovery_client.h
@@ -18,6 +18,7 @@
 #include "base/timer.h"
 
 #include "http/client/http_client.h"
+#include "http/client/http_curl.h"
 
 class ServiceType;
 class DiscoveryServiceClient;
@@ -103,6 +104,7 @@ struct DSPublishResponse {
     int pub_hb_sent_;
     int pub_hb_fail_;
     int pub_hb_rcvd_;
+    int pub_hb_timeout_;
 
     bool publish_cb_called_;
     bool heartbeat_cb_called_;

diff --git a/src/discovery/client/discovery_client_stats.sandesh b/src/discovery/client/discovery_client_stats.sandesh
@@ -15,6 +15,7 @@ struct DiscoveryClientPublisherStats {
     2: u32 heartbeat_sent;
     3: u32 heartbeat_fail;
     4: u32 heartbeat_rcvd;
+    10: u32 heartbeat_timeout;
     5: u32 publish_sent;
     6: u32 publish_fail;
     7: u32 publish_rcvd;