Walk Discovery Publisher list during service unavailability.

On published services being detected unavailable (via TCP close),
walk the list of published services recieved from Discovery Server
instead of resubscribing. This is particularly useful when
cfg-node is inaccessible during upgrade etc.,

Also the list returned by the Discovery Server is an ordered list
based on priority and not equal priority, hence connects should be
ordered.

Use std::rotate to demote the node in the list to the end, this
is more effecient as it solves iterator node deallocate/allocate.

Add additional tests to test border conditions.

Change-Id: Iba6115201cac619a69fa9c92f21d29eb0d729e19
Closes-Bug:1605412

src/vnsw/agent/controller/controller_dns.cc

@@ -112,7 +112,21 @@ void AgentDnsXmppChannel::HandleXmppClientChannelEvent(AgentDnsXmppChannel *peer
     } else if (state == xmps::TIMEDOUT) {
         DiscoveryAgentClient *dac = Agent::GetInstance()->discovery_client();
         if (dac) {
-            dac->ReDiscoverDNS();
+            std::vector<DSResponse> resp =
+                Agent::GetInstance()->GetDiscoveryServerResponseList();
+            std::vector<DSResponse>::iterator iter;
+            for (iter = resp.begin(); iter != resp.end(); iter++) {
+                DSResponse dr = *iter;
+                if (peer->GetXmppServer().compare(
+                    dr.ep.address().to_string()) == 0) {
+
+                    // Add the TIMEDOUT server to the end.
+                    if (iter+1 == resp.end()) break;
+                    std::rotate(iter, iter+1, resp.end());
+                    Agent::GetInstance()->controller()->ApplyDiscoveryDnsXmppServices(resp);
+                    break;
+                }
+            }
         }
     }
 }

src/vnsw/agent/controller/controller_init.cc

@@ -2,6 +2,7 @@
  * Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
  */
 
+#include <algorithm>
 #include "base/logging.h"
 #include "base/timer.h"
 #include "base/contrail_ports.h"
@@ -438,11 +439,14 @@ void VNController::DisConnectControllerIfmapServer(uint8_t idx) {
     agent_->reset_controller_ifmap_xmpp_server(idx);
 }
 
-bool VNController::AgentXmppServerExists(const std::string &server_ip,
-                                         std::vector<DSResponse> resp) {
+bool VNController::AgentXmppServerConnectedExists(
+                                 const std::string &server_ip,
+                                 std::vector<DSResponse> resp) {
 
     std::vector<DSResponse>::iterator iter;
-    for (iter = resp.begin(); iter != resp.end(); iter++) {
+    int8_t count = -1;
+    int8_t min_iter = std::min(static_cast<int>(resp.size()), MAX_XMPP_SERVERS);
+    for (iter = resp.begin(); ++count < min_iter; iter++) {
         DSResponse dr = *iter;
         if (dr.ep.address().to_string().compare(server_ip) == 0) {
             return true;
@@ -462,15 +466,17 @@ bool VNController::ApplyDiscoveryXmppServicesInternal(std::vector<DSResponse> re
     std::vector<DSResponse>::iterator iter;
     int8_t count = -1;
     agent_->UpdateDiscoveryServerResponseList(resp);
-    for (iter = resp.begin(); iter != resp.end(); iter++) {
+
+    /* Apply only MAX_XMPP_SERVERS from list as the list is ordered */
+    int8_t min_iter = std::min(static_cast<int>(resp.size()), MAX_XMPP_SERVERS);
+    for (iter = resp.begin(); ++count < min_iter; iter++) {
         DSResponse dr = *iter;
-        count ++;
 
         CONTROLLER_DISCOVERY_TRACE(DiscoveryConnection, "XMPP Discovery Server Response",
             count, dr.ep.address().to_string(), integerToString(dr.ep.port()));
 
         AgentXmppChannel *chnl = FindAgentXmppChannel(dr.ep.address().to_string());
-        if (chnl) { 
+        if (chnl) {
             if (chnl->GetXmppChannel() &&
                 chnl->GetXmppChannel()->GetPeerState() == xmps::READY) {
                 CONTROLLER_DISCOVERY_TRACE(DiscoveryConnection, 
@@ -502,7 +508,7 @@ bool VNController::ApplyDiscoveryXmppServicesInternal(std::vector<DSResponse> re
 
                 } else if (agent_->controller_xmpp_channel(xs_idx)) {
 
-                    if (AgentXmppServerExists(
+                    if (AgentXmppServerConnectedExists(
                         agent_->controller_ifmap_xmpp_server(xs_idx), resp)) {
 
                         CONTROLLER_DISCOVERY_TRACE(DiscoveryConnection,
@@ -580,16 +586,18 @@ bool VNController::ApplyDiscoveryDnsXmppServicesInternal(
     std::vector<DSResponse>::iterator iter;
     int8_t count = -1;
     agent_->UpdateDiscoveryDnsServerResponseList(resp);
-    for (iter = resp.begin(); iter != resp.end(); iter++) {
+
+    /* Apply only MAX_XMPP_SERVERS from list as the list is ordered */
+    int8_t min_iter = std::min(static_cast<int>(resp.size()), MAX_XMPP_SERVERS);
+    for (iter = resp.begin(); ++count < min_iter; iter++) {
         DSResponse dr = *iter;
-        count++;
 
         CONTROLLER_DISCOVERY_TRACE(DiscoveryConnection,
                                    "DNS Discovery Server Response", count,
             dr.ep.address().to_string(), integerToString(dr.ep.port()));
 
         AgentDnsXmppChannel *chnl = FindAgentDnsXmppChannel(dr.ep.address().to_string());
-        if (chnl) { 
+        if (chnl) {
             if (chnl->GetXmppChannel() &&
                 chnl->GetXmppChannel()->GetPeerState() == xmps::READY) {
                 CONTROLLER_DISCOVERY_TRACE(DiscoveryConnection, 
@@ -619,7 +627,7 @@ bool VNController::ApplyDiscoveryDnsXmppServicesInternal(
 
                 } else if (agent_->dns_xmpp_channel(xs_idx)) {
 
-                    if (AgentXmppServerExists(
+                    if (AgentXmppServerConnectedExists(
                         agent_->dns_server(xs_idx), resp)) {
 
                         CONTROLLER_DISCOVERY_TRACE(DiscoveryConnection,

src/vnsw/agent/controller/controller_init.h

@@ -183,7 +183,7 @@ class VNController {
     AgentDnsXmppChannel *FindAgentDnsXmppChannel(const std::string &server_ip);
     void DeleteConnectionInfo(const std::string &addr, bool is_dns) const;
     const std::string MakeConnectionPrefix(bool is_dns) const;
-    bool AgentXmppServerExists(const std::string &server_ip,
+    bool AgentXmppServerConnectedExists(const std::string &server_ip,
                                std::vector<DSResponse> resp);
     bool  ApplyDiscoveryXmppServicesInternal(std::vector<DSResponse> resp);
     bool  ApplyDiscoveryDnsXmppServicesInternal(std::vector<DSResponse> resp);

src/vnsw/agent/controller/controller_peer.cc

@@ -1612,7 +1612,21 @@ void AgentXmppChannel::HandleAgentXmppClientChannelEvent(AgentXmppChannel *peer,
                          "NULL", "Connection to Xmpp Server, Timed out");
         DiscoveryAgentClient *dac = Agent::GetInstance()->discovery_client();
         if (dac) {
-            dac->ReDiscoverController();
+            std::vector<DSResponse> resp =
+                Agent::GetInstance()->GetDiscoveryServerResponseList();
+            std::vector<DSResponse>::iterator iter;
+            for (iter = resp.begin(); iter != resp.end(); iter++) {
+                DSResponse dr = *iter;
+                if (peer->GetXmppServer().compare(
+                    dr.ep.address().to_string()) == 0) {
+
+                    // Add the TIMEDOUT server to the end.
+                    if (iter+1 == resp.end()) break;
+                    std::rotate(iter, iter+1, resp.end());
+                    agent->controller()->ApplyDiscoveryXmppServices(resp);
+                    break;
+                }
+            }
         }
     }
 }