diff --git a/src/vnsw/agent/oper/agent.sandesh b/src/vnsw/agent/oper/agent.sandesh index fd34676f2b4..727e835ca22 100644 --- a/src/vnsw/agent/oper/agent.sandesh +++ b/src/vnsw/agent/oper/agent.sandesh @@ -1035,6 +1035,7 @@ traceobject sandesh PathPreferenceTrace { 5: string state; 6: i32 retry_timeout; 7: string dependent_ip; + 8: i32 flap_count; } /** diff --git a/src/vnsw/agent/oper/path_preference.cc b/src/vnsw/agent/oper/path_preference.cc index 4559a4f4a1a..75f11747a0d 100644 --- a/src/vnsw/agent/oper/path_preference.cc +++ b/src/vnsw/agent/oper/path_preference.cc @@ -141,9 +141,6 @@ struct TrafficSeen : sc::state { //Enqueue a route change if (state_machine->wait_for_traffic() == true) { state_machine->UpdateFlapTime(); - if (state_machine->flap_count() == 0) { - state_machine->DecreaseRetryTimeout(); - } uint32_t seq = state_machine->max_sequence(); state_machine->set_wait_for_traffic(false); seq++; @@ -252,6 +249,7 @@ PathPreferenceSM::PathPreferenceSM(Agent *agent, const Peer *peer, path_preference_ = pref; initiate(); process_event(EvStart()); + backoff_timer_fired_time_ = UTCTimestampUsec(); } PathPreferenceSM::~PathPreferenceSM() { @@ -271,6 +269,7 @@ PathPreferenceSM::~PathPreferenceSM() { bool PathPreferenceSM::Retry() { flap_count_ = 0; + backoff_timer_fired_time_ = UTCTimestampUsec(); process_event(EvWaitForTraffic()); return false; } @@ -308,30 +307,37 @@ void PathPreferenceSM::IncreaseRetryTimeout() { } } +bool PathPreferenceSM::IsFlap() const { + uint64_t time_sec = (UTCTimestampUsec() - + last_stable_high_priority_change_at_)/1000; + if (time_sec > kMinInterval) { + return false; + } + return true; +} + void PathPreferenceSM::DecreaseRetryTimeout() { - timeout_ = timeout_ / 2; - if (timeout_ < kMinInterval) { + uint64_t time_sec = + (UTCTimestampUsec() - backoff_timer_fired_time_)/1000; + if (time_sec > kMinInterval) { timeout_ = kMinInterval; } } void PathPreferenceSM::UpdateFlapTime() { - uint64_t time_sec = (UTCTimestampUsec() - last_high_priority_change_at_)/1000; - - //Update last flap time - last_high_priority_change_at_ = UTCTimestampUsec(); - if (time_sec < timeout_ + kMinInterval) { + if (IsFlap()) { flap_count_++; } else { + DecreaseRetryTimeout(); + last_stable_high_priority_change_at_ = UTCTimestampUsec(); flap_count_ = 0; } } bool PathPreferenceSM::IsPathFlapping() const { - if (flap_count_ >= kMaxFlapCount) { + if (flap_count_ >= kMaxFlapCount && IsFlap()) { return true; } - return false; } @@ -430,7 +436,7 @@ void PathPreferenceSM::Log(std::string state) { PATH_PREFERENCE_TRACE(rt_->vrf()->GetName(), rt_->GetAddressString(), preference(), sequence(), state, timeout(), - dependent_ip_str); + dependent_ip_str, flap_count_); } void PathPreferenceSM::EnqueuePathChange() { diff --git a/src/vnsw/agent/oper/path_preference.h b/src/vnsw/agent/oper/path_preference.h index adde2475164..747b3654535 100644 --- a/src/vnsw/agent/oper/path_preference.h +++ b/src/vnsw/agent/oper/path_preference.h @@ -29,8 +29,8 @@ class PathPreferenceSM: public sc::state_machine { typedef DependencyList PathDependencyList; public: - static const uint32_t kMinInterval = 5 * 1000; - static const uint32_t kMaxInterval = 100 * 1000; + static const uint32_t kMinInterval = 4 * 1000; + static const uint32_t kMaxInterval = 32 * 1000; static const uint32_t kMaxFlapCount = 5; PathPreferenceSM(Agent *agent, const Peer *peer, AgentRoute *rt, bool dependent_rt, @@ -42,8 +42,8 @@ class PathPreferenceSM: bool ecmp() const {return path_preference_.ecmp();} uint32_t timeout() const { return timeout_;} - uint64_t last_high_priority_change_at() const { - return last_high_priority_change_at_; + uint64_t last_stable_high_priority_change_at() const { + return last_stable_high_priority_change_at_; } uint32_t flap_count() const { return flap_count_;} @@ -76,8 +76,8 @@ class PathPreferenceSM: timeout_ = timeout; } - void set_last_high_priority_change_at(uint64_t timestamp) { - last_high_priority_change_at_ = timestamp; + void set_last_stable_high_priority_change_at(uint64_t timestamp) { + last_stable_high_priority_change_at_ = timestamp; } void set_dependent_rt(PathPreferenceSM *sm) { @@ -96,6 +96,7 @@ class PathPreferenceSM: return path_preference_.dependent_ip(); } + bool IsFlap() const; bool seen() { return seen_; } uint32_t max_sequence() const { return max_sequence_;} void Process(); @@ -121,7 +122,8 @@ class PathPreferenceSM: bool seen_; Timer *timer_; uint32_t timeout_; - uint64_t last_high_priority_change_at_; + uint64_t last_stable_high_priority_change_at_; + uint64_t backoff_timer_fired_time_; uint32_t flap_count_; bool is_dependent_rt_; DependencyRef dependent_rt_; diff --git a/src/vnsw/agent/oper/test/test_aap.cc b/src/vnsw/agent/oper/test/test_aap.cc index aae1e1a15ec..dc814f4e49a 100644 --- a/src/vnsw/agent/oper/test/test_aap.cc +++ b/src/vnsw/agent/oper/test/test_aap.cc @@ -84,6 +84,7 @@ class TestAap : public ::testing::Test { EXPECT_TRUE(VmPortActive(1)); AddIPAM("vn1", ipam_info, 2); client->WaitForIdle(); + seq_no = 1; } virtual void TearDown() { @@ -95,9 +96,40 @@ class TestAap : public ::testing::Test { DelIPAM("vn1"); client->WaitForIdle(); } + + void FlapRoute(const Ip4Address ip, const VmInterface *vm_intf, + uint32_t flap_count) { + InetUnicastRouteEntry *rt = RouteGet("vrf1", ip, 32); + const AgentPath *path = rt->FindPath(vm_intf->peer()); + + for (uint32_t i = 0; i <= flap_count; i++) { + Agent::GetInstance()->oper_db()->route_preference_module()-> + EnqueueTrafficSeen(ip, 32, vm_intf->id(), vm_intf->vrf()->vrf_id(), + vm_intf->vm_mac()); + client->WaitForIdle(); + EXPECT_TRUE(path->path_preference().sequence() == seq_no++); + EXPECT_TRUE(path->path_preference().preference() == PathPreference::HIGH); + EXPECT_TRUE(path->path_preference().ecmp() == false); + EXPECT_TRUE(path->path_preference().wait_for_traffic() == false); + + Ip4Address server_ip = Ip4Address::from_string("10.1.1.3"); + TunnelType::TypeBmap bmap = (1 << TunnelType::MPLS_GRE); + PathPreference path_preference(seq_no++, PathPreference::HIGH, false, false); + Inet4TunnelRouteAdd(peer_, "vrf1", ip, 32, server_ip, bmap, + 16, "vn1", SecurityGroupList(), path_preference); + client->WaitForIdle(); + if (i < PathPreferenceSM::kMaxFlapCount) { + EXPECT_TRUE(path->path_preference().preference() == PathPreference::LOW); + EXPECT_TRUE(path->path_preference().ecmp() == false); + EXPECT_TRUE(path->path_preference().wait_for_traffic() == true); + } + } + } + protected: Peer *peer_; Agent *agent_; + uint32_t seq_no; }; //Add and delete allowed address pair route @@ -759,143 +791,120 @@ TEST_F(TestAap, StateMachine_13) { client->WaitForIdle(); } -TEST_F(TestAap, StateMachine_14) { +TEST_F(TestAap, Backoff_1) { Ip4Address ip = Ip4Address::from_string("1.1.1.1"); - Ip4Address server_ip = Ip4Address::from_string("10.1.1.3"); VmInterface *vm_intf = VmInterfaceGet(1); InetUnicastRouteEntry *rt = RouteGet("vrf1", ip, 32); const AgentPath *path = rt->FindPath(vm_intf->peer()); - uint32_t seq_no = 1; - for (uint32_t i = 0 ; i <= PathPreferenceSM::kMaxFlapCount; i++) { - Agent::GetInstance()->oper_db()->route_preference_module()-> - EnqueueTrafficSeen(ip, 32, vm_intf->id(), vm_intf->vrf()->vrf_id(), - vm_intf->vm_mac()); - client->WaitForIdle(); - EXPECT_TRUE(path->path_preference().sequence() == seq_no++); - EXPECT_TRUE(path->path_preference().preference() == PathPreference::HIGH); - EXPECT_TRUE(path->path_preference().ecmp() == false); - EXPECT_TRUE(path->path_preference().wait_for_traffic() == false); - - TunnelType::TypeBmap bmap = (1 << TunnelType::MPLS_GRE); - PathPreference path_preference(seq_no++, PathPreference::HIGH, false, false); - Inet4TunnelRouteAdd(peer_, "vrf1", ip, 32, server_ip, bmap, - 16, "vn1", SecurityGroupList(), path_preference); - client->WaitForIdle(); + FlapRoute(ip, vm_intf, PathPreferenceSM::kMaxFlapCount); + EXPECT_TRUE(path->path_preference().preference() == PathPreference::HIGH); + //Timeout has increase to 8 second + usleep(2 * 1000 * (PathPreferenceSM::kMinInterval)); + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); - if (i != PathPreferenceSM::kMaxFlapCount) { - EXPECT_TRUE(path->path_preference().preference() == - PathPreference::LOW); - EXPECT_TRUE(path->path_preference().ecmp() == false); - EXPECT_TRUE(path->path_preference().wait_for_traffic() == true); - } - } + FlapRoute(ip, vm_intf, PathPreferenceSM::kMaxFlapCount); + //Timeout increases to 16 sec + usleep(2 * 1000 * (PathPreferenceSM::kMinInterval)); + //Path should be in wait for traffic after 8 second sleep + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == false)); - EXPECT_TRUE(path->path_preference().preference() == PathPreference::HIGH); + //16 second sleep over, path goes to wait for traffic state usleep(2 * 1000 * (PathPreferenceSM::kMinInterval)); - //Check that agent withdraws its route after 100ms - //since BGP path, didnt reflect local agent nexthop WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); + client->WaitForIdle(); +} - for (uint32_t i = 0; i <= PathPreferenceSM::kMaxFlapCount - 1; i++) { - Agent::GetInstance()->oper_db()->route_preference_module()-> - EnqueueTrafficSeen(ip, 32, vm_intf->id(), vm_intf->vrf()->vrf_id(), - vm_intf->vm_mac()); - client->WaitForIdle(); - EXPECT_TRUE(path->path_preference().sequence() == seq_no++); - EXPECT_TRUE(path->path_preference().preference() == PathPreference::HIGH); - EXPECT_TRUE(path->path_preference().ecmp() == false); - EXPECT_TRUE(path->path_preference().wait_for_traffic() == false); +//Check that if route flap stops, timeout decreases +TEST_F(TestAap, Backoff_2) { + Ip4Address ip = Ip4Address::from_string("1.1.1.1"); + VmInterface *vm_intf = VmInterfaceGet(1); + InetUnicastRouteEntry *rt = + RouteGet("vrf1", ip, 32); + const AgentPath *path = rt->FindPath(vm_intf->peer()); - TunnelType::TypeBmap bmap = (1 << TunnelType::MPLS_GRE); - PathPreference path_preference(seq_no++, PathPreference::HIGH, false, false); - Inet4TunnelRouteAdd(peer_, "vrf1", ip, 32, server_ip, bmap, - 16, "vn1", SecurityGroupList(), path_preference); - client->WaitForIdle(); - if (i != PathPreferenceSM::kMaxFlapCount - 1) { - EXPECT_TRUE(path->path_preference().preference() == PathPreference::LOW); - EXPECT_TRUE(path->path_preference().ecmp() == false); - EXPECT_TRUE(path->path_preference().wait_for_traffic() == true); - } - } + FlapRoute(ip, vm_intf, PathPreferenceSM::kMaxFlapCount); + //Timeout is 8 seconds + //Sleep for 16 seconds so that route flap stops + usleep(4 * 1000 * (PathPreferenceSM::kMinInterval)); + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); + client->WaitForIdle(); - //Timeout increases to 20 sec + FlapRoute(ip, vm_intf, PathPreferenceSM::kMaxFlapCount); + //Timeout : 8s + //Sleep for : 8s usleep(2 * 1000 * (PathPreferenceSM::kMinInterval)); //Check that agent withdraws its route after 100ms //since BGP path, didnt reflect local agent nexthop - WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == false)); + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); +} + +TEST_F(TestAap, Backoff_3) { + Ip4Address ip = Ip4Address::from_string("1.1.1.1"); + VmInterface *vm_intf = VmInterfaceGet(1); + InetUnicastRouteEntry *rt = RouteGet("vrf1", ip, 32); + const AgentPath *path = rt->FindPath(vm_intf->peer()); + + //No continous flaps + FlapRoute(ip, vm_intf, 4); + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); + //Dummy sleep usleep(2 * 1000 * (PathPreferenceSM::kMinInterval)); WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); client->WaitForIdle(); } -//Check that if route flap stops, timeout decreases -TEST_F(TestAap, StateMachine_15) { +//Flap for 4 times, so that max back off is reached +//Sleep for 4 second of inactivity, verify path moves back to 4 second backoff +TEST_F(TestAap, Backoff_4) { Ip4Address ip = Ip4Address::from_string("1.1.1.1"); - Ip4Address server_ip = Ip4Address::from_string("10.1.1.3"); VmInterface *vm_intf = VmInterfaceGet(1); InetUnicastRouteEntry *rt = RouteGet("vrf1", ip, 32); const AgentPath *path = rt->FindPath(vm_intf->peer()); - uint32_t seq_no = 1; - for (uint32_t i = 0 ; i <= PathPreferenceSM::kMaxFlapCount; i++) { - Agent::GetInstance()->oper_db()->route_preference_module()-> - EnqueueTrafficSeen(ip, 32, vm_intf->id(), vm_intf->vrf()->vrf_id(), - vm_intf->vm_mac()); + for (uint32_t i = 1; i <= 3; i++) { + FlapRoute(ip, vm_intf, PathPreferenceSM::kMaxFlapCount); + uint32_t backoff = 1 << i; + usleep(backoff * 1000 * (PathPreferenceSM::kMinInterval)); + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); client->WaitForIdle(); - EXPECT_TRUE(path->path_preference().sequence() == seq_no++); - EXPECT_TRUE(path->path_preference().preference() == PathPreference::HIGH); - EXPECT_TRUE(path->path_preference().ecmp() == false); - EXPECT_TRUE(path->path_preference().wait_for_traffic() == false); - - TunnelType::TypeBmap bmap = (1 << TunnelType::MPLS_GRE); - PathPreference path_preference(seq_no++, PathPreference::HIGH, false, false); - Inet4TunnelRouteAdd(peer_, "vrf1", ip, 32, server_ip, bmap, - 16, "vn1", SecurityGroupList(), path_preference); - client->WaitForIdle(); - if (i != PathPreferenceSM::kMaxFlapCount) { - EXPECT_TRUE(path->path_preference().preference() == - PathPreference::LOW); - EXPECT_TRUE(path->path_preference().ecmp() == false); - EXPECT_TRUE(path->path_preference().wait_for_traffic() == true); - } } - //Sleep for 40 seconds so that route flap stops - usleep(4 * 1000 * (PathPreferenceSM::kMinInterval)); + //No flap for 4s, hence backoff should be reset to 4s + usleep((PathPreferenceSM::kMinInterval) * 1000); + FlapRoute(ip, vm_intf, PathPreferenceSM::kMaxFlapCount); + usleep(2 * 1000 * (PathPreferenceSM::kMinInterval)); WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); - client->WaitForIdle(); +} - for (uint32_t i = 0; i <= PathPreferenceSM::kMaxFlapCount; i++) { - Agent::GetInstance()->oper_db()->route_preference_module()-> - EnqueueTrafficSeen(ip, 32, vm_intf->id(), vm_intf->vrf()->vrf_id(), - vm_intf->vm_mac()); - client->WaitForIdle(); - EXPECT_TRUE(path->path_preference().sequence() == seq_no++); - EXPECT_TRUE(path->path_preference().preference() == PathPreference::HIGH); - EXPECT_TRUE(path->path_preference().ecmp() == false); - EXPECT_TRUE(path->path_preference().wait_for_traffic() == false); +//Ensure backoff time continous at 32 seconds +//given route keeps on flapping after every backoff timer firing +TEST_F(TestAap, Backoff_5) { + Ip4Address ip = Ip4Address::from_string("1.1.1.1"); + VmInterface *vm_intf = VmInterfaceGet(1); + InetUnicastRouteEntry *rt = + RouteGet("vrf1", ip, 32); + const AgentPath *path = rt->FindPath(vm_intf->peer()); - TunnelType::TypeBmap bmap = (1 << TunnelType::MPLS_GRE); - PathPreference path_preference(seq_no++, PathPreference::HIGH, false, false); - Inet4TunnelRouteAdd(peer_, "vrf1", ip, 32, server_ip, bmap, - 16, "vn1", SecurityGroupList(), path_preference); + for (uint32_t i = 1; i <= 3; i++) { + FlapRoute(ip, vm_intf, PathPreferenceSM::kMaxFlapCount); + uint32_t backoff = 1 << i; + usleep(backoff * 1000 * (PathPreferenceSM::kMinInterval)); + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); client->WaitForIdle(); - if (i < PathPreferenceSM::kMaxFlapCount) { - EXPECT_TRUE(path->path_preference().preference() == - PathPreference::LOW); - EXPECT_TRUE(path->path_preference().ecmp() == false); - EXPECT_TRUE(path->path_preference().wait_for_traffic() == true); - } } - EXPECT_TRUE(path->path_preference().wait_for_traffic() == false); - //Timeout increases to 20 sec + + FlapRoute(ip, vm_intf, PathPreferenceSM::kMaxFlapCount); + usleep(1000 * (PathPreferenceSM::kMinInterval)); + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == false)); usleep(2 * 1000 * (PathPreferenceSM::kMinInterval)); - //Check that agent withdraws its route after 100ms - //since BGP path, didnt reflect local agent nexthop + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == false)); + usleep(4 * 1000 * (PathPreferenceSM::kMinInterval)); + WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == false)); + usleep(1000 * (PathPreferenceSM::kMinInterval)); WAIT_FOR(1000, 1000, (path->path_preference().wait_for_traffic() == true)); }