diff --git a/src/bgp/bgp_membership.cc b/src/bgp/bgp_membership.cc index 891d9e6c1bb..3dc358c036a 100644 --- a/src/bgp/bgp_membership.cc +++ b/src/bgp/bgp_membership.cc @@ -141,8 +141,7 @@ void BgpMembershipManager::RegisterRibIn(IPeer *peer, BgpTable *table) { // Post an UNREGISTER_RIB event to deal with concurrency issues with RibOut. // void BgpMembershipManager::Unregister(IPeer *peer, BgpTable *table) { - CHECK_CONCURRENCY("bgp::Config", "bgp::StateMachine", "xmpp::StateMachine", - "bgp::PeerMembership"); + CHECK_CONCURRENCY("bgp::Config", "bgp::StateMachine", "xmpp::StateMachine"); tbb::spin_rw_mutex::scoped_lock write_lock(rw_mutex_, true); current_jobs_count_++; @@ -168,8 +167,7 @@ void BgpMembershipManager::Unregister(IPeer *peer, BgpTable *table) { // Unregister the IPeer from the BgpTable for RIBIN. // void BgpMembershipManager::UnregisterRibIn(IPeer *peer, BgpTable *table) { - CHECK_CONCURRENCY("bgp::Config", "bgp::StateMachine", "xmpp::StateMachine", - "bgp::PeerMembership"); + CHECK_CONCURRENCY("bgp::Config", "bgp::StateMachine", "xmpp::StateMachine"); tbb::spin_rw_mutex::scoped_lock write_lock(rw_mutex_, true); current_jobs_count_++; @@ -201,8 +199,7 @@ void BgpMembershipManager::UnregisterRibInUnlocked(PeerRibState *prs) { // This API is to be used when handling graceful restart of the peer. // void BgpMembershipManager::UnregisterRibOut(IPeer *peer, BgpTable *table) { - CHECK_CONCURRENCY("bgp::Config", "bgp::StateMachine", "xmpp::StateMachine", - "bgp::PeerMembership"); + CHECK_CONCURRENCY("bgp::Config", "bgp::StateMachine", "xmpp::StateMachine"); tbb::spin_rw_mutex::scoped_lock write_lock(rw_mutex_, true); current_jobs_count_++; @@ -224,8 +221,7 @@ void BgpMembershipManager::UnregisterRibOut(IPeer *peer, BgpTable *table) { // It can also be used in future when re-evaluating import policy for a peer. // void BgpMembershipManager::WalkRibIn(IPeer *peer, BgpTable *table) { - CHECK_CONCURRENCY("bgp::Config", "bgp::StateMachine", "xmpp::StateMachine", - "bgp::PeerMembership"); + CHECK_CONCURRENCY("bgp::Config", "bgp::StateMachine", "xmpp::StateMachine"); tbb::spin_rw_mutex::scoped_lock write_lock(rw_mutex_, true); current_jobs_count_++; diff --git a/src/bgp/bgp_peer.cc b/src/bgp/bgp_peer.cc index ec7f25741f4..ef0f6f9feda 100644 --- a/src/bgp/bgp_peer.cc +++ b/src/bgp/bgp_peer.cc @@ -115,6 +115,8 @@ class BgpPeer::PeerClose : public IPeerClose { peer_->ReceiveEndOfRIB(family, 0); } + virtual void MembershipRequestCallbackComplete(bool result) { } + bool IsGRReady() const { // Check if GR helper mode is disabled. if (!peer_->server()->gr_helper_enable()) @@ -323,6 +325,8 @@ class BgpPeer::DeleteActor : public LifetimeActor { virtual bool MayDelete() const { CHECK_CONCURRENCY("bgp::Config"); + if (!peer_->peer_close_->close_manager()->IsQueueEmpty()) + return false; if (peer_->IsCloseInProgress()) return false; if (!peer_->state_machine_->IsQueueEmpty()) @@ -344,8 +348,7 @@ class BgpPeer::DeleteActor : public LifetimeActor { peer_->server()->decrement_deleting_count(); } assert(!peer_->membership_req_pending()); - assert(peer_->peer_close()->close_manager()->membership_state() != - PeerCloseManager::MEMBERSHIP_IN_USE); + assert(!peer_->peer_close_->close_manager()->IsMembershipInUse()); peer_->rtinstance_->peer_manager()->DestroyIPeer(peer_); } @@ -490,9 +493,8 @@ bool BgpPeer::CanUseMembershipManager() const { // in question. // void BgpPeer::MembershipRequestCallback(BgpTable *table) { - if (peer_close_->close_manager()->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_USE) { - (void) peer_close_->close_manager()->MembershipRequestCallback(); + if (peer_close_->close_manager()->IsMembershipInUse()) { + peer_close_->close_manager()->MembershipRequestCallback(); return; } @@ -501,8 +503,7 @@ void BgpPeer::MembershipRequestCallback(BgpTable *table) { // Resume if CloseManager is waiting to use membership manager. if (!membership_req_pending_ && - peer_close_->close_manager()->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_WAIT) { + peer_close_->close_manager()->IsMembershipInWait()) { peer_close_->close_manager()->MembershipRequest(); } @@ -1037,8 +1038,7 @@ void BgpPeer::CustomClose() { // void BgpPeer::Close(bool non_graceful) { if (membership_req_pending_ && - peer_close_->close_manager()->membership_state() != - PeerCloseManager::MEMBERSHIP_IN_USE) { + !peer_close_->close_manager()->IsMembershipInUse()) { BGP_LOG_PEER(Event, this, SandeshLevel::SYS_INFO, BGP_LOG_FLAG_ALL, BGP_PEER_DIR_NA, "Close procedure deferred"); defer_close_ = true; @@ -1134,10 +1134,8 @@ bool BgpPeer::AcceptSession(BgpSession *session) { } void BgpPeer::Register(BgpTable *table, const RibExportPolicy &policy) { - assert(peer_close_->close_manager()->membership_state() != - PeerCloseManager::MEMBERSHIP_IN_USE); - if (peer_close_->close_manager()->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_WAIT) + assert(!peer_close_->close_manager()->IsMembershipInUse()); + if (peer_close_->close_manager()->IsMembershipInWait()) assert(membership_req_pending_ > 0); BgpMembershipManager *membership_mgr = server_->membership_mgr(); membership_req_pending_++; @@ -1145,10 +1143,8 @@ void BgpPeer::Register(BgpTable *table, const RibExportPolicy &policy) { } void BgpPeer::Register(BgpTable *table) { - assert(peer_close_->close_manager()->membership_state() != - PeerCloseManager::MEMBERSHIP_IN_USE); - if (peer_close_->close_manager()->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_WAIT) + assert(!peer_close_->close_manager()->IsMembershipInUse()); + if (peer_close_->close_manager()->IsMembershipInWait()) assert(membership_req_pending_ > 0); BgpMembershipManager *membership_mgr = server_->membership_mgr(); membership_mgr->RegisterRibIn(this, table); @@ -1649,10 +1645,8 @@ bool BgpPeer::SetGRCapabilities(BgpPeerInfoData *peer_info) { // If GR is no longer supported, terminate GR right away. This can happen // due to mis-match between gr and llgr afis. For now, we expect an // identical set. - if ((peer_close_->close_manager()->state() == PeerCloseManager::GR_TIMER && - !peer_close_->IsCloseGraceful()) || - (peer_close_->close_manager()->state() == - PeerCloseManager::LLGR_TIMER && + if (peer_close_->close_manager()->IsInGracefulRestartTimerWait() && + (!peer_close_->IsCloseGraceful() || !peer_close_->IsCloseLongLivedGraceful())) { Close(true); return false; diff --git a/src/bgp/bgp_peer_close.cc b/src/bgp/bgp_peer_close.cc index a7977ba2cdd..843202b7472 100644 --- a/src/bgp/bgp_peer_close.cc +++ b/src/bgp/bgp_peer_close.cc @@ -44,51 +44,38 @@ // Create an instance of PeerCloseManager with back reference to parent IPeer PeerCloseManager::PeerCloseManager(IPeerClose *peer_close, boost::asio::io_service &io_service) : - peer_close_(peer_close), stale_timer_(NULL), sweep_timer_(NULL), - stale_notify_timer_(NULL), + peer_close_(peer_close), stale_timer_(NULL), + event_queue_(new WorkQueue( + TaskScheduler::GetInstance()->GetTaskId("bgp::Config"), 0, + boost::bind(&PeerCloseManager::EventCallback, this, _1))), state_(NONE), close_again_(false), non_graceful_(false), gr_elapsed_(0), llgr_elapsed_(0), membership_state_(MEMBERSHIP_NONE) { stats_.init++; membership_req_pending_ = 0; stale_timer_ = TimerManager::CreateTimer(io_service, - "Graceful Restart StaleTimer", - TaskScheduler::GetInstance()->GetTaskId("bgp::Config"), 0); - sweep_timer_ = TimerManager::CreateTimer(io_service, - "Graceful Restart SweepTimer", - TaskScheduler::GetInstance()->GetTaskId("bgp::Config"), 0); - stale_notify_timer_ = TimerManager::CreateTimer(io_service, - "Graceful Restart StaleNotifyTimer", - TaskScheduler::GetInstance()->GetTaskId("bgp::Config"), 0); + "Graceful Restart StaleTimer"); } // Create an instance of PeerCloseManager with back reference to parent IPeer PeerCloseManager::PeerCloseManager(IPeerClose *peer_close) : - peer_close_(peer_close), stale_timer_(NULL), sweep_timer_(NULL), - stale_notify_timer_(NULL), + peer_close_(peer_close), stale_timer_(NULL), + event_queue_(new WorkQueue( + TaskScheduler::GetInstance()->GetTaskId("bgp::Config"), 0, + boost::bind(&PeerCloseManager::EventCallback, this, _1))), state_(NONE), close_again_(false), non_graceful_(false), gr_elapsed_(0), llgr_elapsed_(0), membership_state_(MEMBERSHIP_NONE) { stats_.init++; membership_req_pending_ = 0; if (peer_close->peer() && peer_close->peer()->server()) { - stale_timer_ = TimerManager::CreateTimer( - *peer_close->peer()->server()->ioservice(), - "Graceful Restart StaleTimer", - TaskScheduler::GetInstance()->GetTaskId("bgp::Config"), 0); - sweep_timer_ = TimerManager::CreateTimer( - *peer_close->peer()->server()->ioservice(), - "Graceful Restart SweepTimer", - TaskScheduler::GetInstance()->GetTaskId("bgp::Config"), 0); - stale_notify_timer_ = TimerManager::CreateTimer( - *peer_close->peer()->server()->ioservice(), - "Graceful Restart StaleNotifyTimer", - TaskScheduler::GetInstance()->GetTaskId("bgp::Config"), 0); + stale_timer_ = + TimerManager::CreateTimer(*peer_close->peer()->server()->ioservice(), + "Graceful Restart StaleTimer"); } } PeerCloseManager::~PeerCloseManager() { + event_queue_->Shutdown(); TimerManager::DeleteTimer(stale_timer_); - TimerManager::DeleteTimer(sweep_timer_); - TimerManager::DeleteTimer(stale_notify_timer_); } std::string PeerCloseManager::GetStateName(State state) const { @@ -134,23 +121,23 @@ std::string PeerCloseManager::GetMembershipStateName( // // Peer IsReady() in GR timer callback (or via reception of all EoRs) // RibIn Sweep and Ribout Generation close_state_: SWEEP -// MembershipRequestCallback close_state_: NONE +// MembershipRequestCallback close_state_: NONE // // Peer not IsReady() in GR timer callback // If LLGR supported close_state_: LLGR_STALE // RibIn Stale marking with LLGR_STALE community close_state_: LLGR_TIMER // // Peer not IsReady() in LLGR timer callback -// RibIn Delete close_state_: DELETE +// RibIn Delete close_state_: DELETE // MembershipRequestCallback close_state_: NONE // // Peer IsReady() in LLGR timer callback (or via reception of all EoRs) -// RibIn Sweep close_state_: SWEEP -// MembershipRequestCallback close_state_: NONE +// RibIn Sweep close_state_: SWEEP +// MembershipRequestCallback close_state_: NONE // // If LLGR is not supported -// RibIn Delete close_state_: DELETE -// MembershipRequestCallback close_state_: NONE +// RibIn Delete close_state_: DELETE +// MembershipRequestCallback close_state_: NONE // // Close() call during any state other than NONE and DELETE // Cancel GR timer and restart GR Closure all over again @@ -164,12 +151,15 @@ std::string PeerCloseManager::GetMembershipStateName( // // Use non_graceful as true for non-graceful closure void PeerCloseManager::Close(bool non_graceful) { - tbb::mutex::scoped_lock lock(mutex_); + EnqueueEvent(new Event(CLOSE, non_graceful)); +} + +void PeerCloseManager::Close(Event *event) { // Note down non-graceful close trigger. Once non-graceful closure is // triggered, it should remain so until close process is complete. Further // graceful closure calls until then should remain non-graceful. - non_graceful_ |= non_graceful; + non_graceful_ |= event->non_graceful; CloseInternal(); } @@ -214,20 +204,13 @@ void PeerCloseManager::CloseInternal() { } } -bool PeerCloseManager::GRTimerFired() const { - return stale_timer_->fired(); -} - void PeerCloseManager::ProcessEORMarkerReceived(Address::Family family) { - if (GRTimerFired()) { - assert(state_ == GR_TIMER || state_ == LLGR_TIMER); - families_.erase(family); - return; - } + EnqueueEvent(new Event(EOR_RECEIVED, family)); +} - tbb::mutex::scoped_lock lock(mutex_); +void PeerCloseManager::ProcessEORMarkerReceived(Event *event) { if ((state_ == GR_TIMER || state_ == LLGR_TIMER) && !families_.empty()) { - families_.erase(family); + families_.erase(event->family); // Start the timer if all EORs have been received. if (families_.empty()) @@ -246,14 +229,20 @@ void PeerCloseManager::StartRestartTimer(int time) { } bool PeerCloseManager::RestartTimerCallback() { + CHECK_CONCURRENCY("timer::TimerTask"); + EnqueueEvent(new Event(TIMER_CALLBACK)); + return false; +} + +void PeerCloseManager::RestartTimerCallback(Event *event) { CHECK_CONCURRENCY("bgp::Config"); - tbb::mutex::scoped_lock lock(mutex_); PEER_CLOSE_MANAGER_LOG("GR Timer callback started"); if (state_ != GR_TIMER && state_ != LLGR_TIMER) - return false; + return; if (peer_close_->IsReady() && !families_.empty()) { + // Fake reception of all EORs. for (IPeerClose::Families::iterator i = families_.begin(), next = i; i != families_.end(); i = next) { @@ -261,13 +250,9 @@ bool PeerCloseManager::RestartTimerCallback() { PEER_CLOSE_MANAGER_LOG("Simulate EoR reception for family " << *i); peer_close_->ReceiveEndOfRIB(*i); } - - // Restart the timer to fire right away. - stale_timer_->Reschedule(0); - return true; + } else { + ProcessClosure(); } - ProcessClosure(); - return false; } // Route stale timer callback. If the peer has come back up, sweep routes for @@ -327,13 +312,12 @@ void PeerCloseManager::ProcessClosure() { if (state_ == DELETE) peer_close_->CustomClose(); - MembershipRequestInternal(); + MembershipRequest(); } void PeerCloseManager::CloseComplete() { MOVE_TO_STATE(NONE); stale_timer_->Cancel(); - sweep_timer_->Cancel(); families_.clear(); stats_.init++; @@ -344,43 +328,35 @@ void PeerCloseManager::CloseComplete() { } } -bool PeerCloseManager::ProcessSweepStateActions() { +void PeerCloseManager::TriggerSweepStateActions() { CHECK_CONCURRENCY("bgp::Config"); assert(state_ == SWEEP); // Notify clients to trigger sweep as appropriate. peer_close_->GracefulRestartSweep(); CloseComplete(); - return false; -} - -void PeerCloseManager::TriggerSweepStateActions() { - PEER_CLOSE_MANAGER_LOG("Sweep Timer started to fire right away"); - sweep_timer_->Cancel(); - sweep_timer_->Start(0, - boost::bind(&PeerCloseManager::ProcessSweepStateActions, this)); -} - -bool PeerCloseManager::NotifyStaleEvent() { - CHECK_CONCURRENCY("bgp::Config"); - - peer_close_->GracefulRestartStale(); - MembershipRequestInternal(); - return false; } // Notify clients about entering Stale event. Do this through a timer callback // off bgp::Config to solve concurrency issues in the client. void PeerCloseManager::StaleNotify() { - PEER_CLOSE_MANAGER_LOG("Stale Notify Timer started to fire right away"); - stale_notify_timer_->Cancel(); - stale_notify_timer_->Start(0, - boost::bind(&PeerCloseManager::NotifyStaleEvent, this)); + CHECK_CONCURRENCY("bgp::Config"); + + peer_close_->GracefulRestartStale(); + MembershipRequest(NULL); } void PeerCloseManager::MembershipRequest() { - tbb::mutex::scoped_lock lock(mutex_); - MembershipRequestInternal(); + assert(membership_state_ != MEMBERSHIP_IN_USE); + + // Pause if membership manager is not ready for usage. + if (!CanUseMembershipManager()) { + set_membership_state(MEMBERSHIP_IN_WAIT); + PEER_CLOSE_MANAGER_LOG("Wait for membership manager availability"); + return; + } + set_membership_state(MEMBERSHIP_IN_USE); + EnqueueEvent(new Event(MEMBERSHIP_REQUEST)); } bool PeerCloseManager::CanUseMembershipManager() const { @@ -391,27 +367,20 @@ BgpMembershipManager *PeerCloseManager::membership_mgr() const { return peer_close_->peer()->server()->membership_mgr(); } -void PeerCloseManager::MembershipRequestInternal() { - assert(membership_state() != MEMBERSHIP_IN_USE); +void PeerCloseManager::MembershipRequest(Event *evnet) { + CHECK_CONCURRENCY("bgp::Config"); - // Pause if membership manager is not ready for usage. - if (!CanUseMembershipManager()) { - set_membership_state(MEMBERSHIP_IN_WAIT); - PEER_CLOSE_MANAGER_LOG("Wait for membership manager availability"); - return; - } set_membership_state(MEMBERSHIP_IN_USE); assert(!membership_req_pending_); membership_req_pending_++; BgpMembershipManager *mgr = membership_mgr(); if (!mgr) return; - std::list tables; mgr->GetRegisteredRibs(peer_close_->peer(), &tables); if (tables.empty()) { - assert(MembershipRequestCompleteCallbackInternal()); + assert(MembershipRequestCallback(NULL)); return; } @@ -446,30 +415,31 @@ void PeerCloseManager::MembershipRequestInternal() { } } -// Concurrency: Runs in the context of the BGP peer rib membership task. +void PeerCloseManager::MembershipRequestCallback() { + EnqueueEvent(new Event(MEMBERSHIP_REQUEST_COMPLETE_CALLBACK)); +} + +// Concurrency: Runs in the context of the bgp::Config task // // Close process for this peer in terms of walking RibIns and RibOuts are // complete. Do the final cleanups necessary and notify interested party // // Retrun true if we are done using membership manager, false otherwise. -bool PeerCloseManager::MembershipRequestCallback() { - tbb::mutex::scoped_lock lock(mutex_); - return MembershipRequestCompleteCallbackInternal(); -} +bool PeerCloseManager::MembershipRequestCallback(Event *event) { + CHECK_CONCURRENCY("bgp::Config"); -bool PeerCloseManager::MembershipRequestCompleteCallbackInternal() { assert(state_ == STALE || LLGR_STALE || state_ == SWEEP || state_ == DELETE); - assert(membership_state() == MEMBERSHIP_IN_USE); + assert(membership_state_ == MEMBERSHIP_IN_USE); assert(membership_req_pending_ > 0); PEER_CLOSE_MANAGER_LOG("MembershipRequestCallback"); - bool ret = false; + bool result = false; if (--membership_req_pending_) - return ret; + return result; // Indicate to the caller that we are done using the membership manager. - ret = true; + result = true; set_membership_state(MEMBERSHIP_NONE); if (state_ == DELETE) { @@ -480,13 +450,13 @@ bool PeerCloseManager::MembershipRequestCompleteCallbackInternal() { stats_.init++; close_again_ = false; non_graceful_ = false; - return ret; + return result; } // Process nested closures. if (close_again_) { CloseComplete(); - return ret; + return result; } // If any GR stale timer has to be launched, then to wait for some time @@ -503,7 +473,7 @@ bool PeerCloseManager::MembershipRequestCompleteCallbackInternal() { time = 0; StartRestartTimer(time); stats_.gr_timer++; - return ret; + return result; } // From LLGR_STALE state, switch to LLGR_TIMER state. Typically this would @@ -516,22 +486,20 @@ bool PeerCloseManager::MembershipRequestCompleteCallbackInternal() { peer_close_->GetLongLivedGracefulRestartTime()); // Offset restart time with elapsed time during nested closures. - int time = peer_close_->GetLongLivedGracefulRestartTime() *1000; + int time = peer_close_->GetLongLivedGracefulRestartTime() * 1000; time -= llgr_elapsed_; if (time < 0) time = 0; StartRestartTimer(time); stats_.llgr_timer++; - return ret; + return result; } TriggerSweepStateActions(); - return ret; + return result; } void PeerCloseManager::FillCloseInfo(BgpNeighborResp *resp) const { - tbb::mutex::scoped_lock lock(mutex_); - PeerCloseInfo peer_close_info; peer_close_info.state = GetStateName(state_); peer_close_info.membership_state = @@ -553,6 +521,7 @@ void PeerCloseManager::FillCloseInfo(BgpNeighborResp *resp) const { bool PeerCloseManager::MembershipPathCallback(DBTablePartBase *root, BgpRoute *rt, BgpPath *path) { + CHECK_CONCURRENCY("db::DBTable"); DBRequest::DBOperation oper; BgpAttrPtr attrs; @@ -628,3 +597,37 @@ bool PeerCloseManager::MembershipPathCallback(DBTablePartBase *root, attrs, path->GetPathId(), path->GetFlags() | stale, path->GetLabel()); } + +// +// Handler for an Event. +// +bool PeerCloseManager::EventCallback(Event *event) { + CHECK_CONCURRENCY("bgp::Config"); + bool result; + + switch (event->event_type) { + case EVENT_NONE: + break; + case CLOSE: + Close(event); + break; + case EOR_RECEIVED: + ProcessEORMarkerReceived(event); + break; + case MEMBERSHIP_REQUEST: + MembershipRequest(event); + break; + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + result = MembershipRequestCallback(event); + + // Notify clients with the result of this processing. + peer_close_->MembershipRequestCallbackComplete(result); + break; + case TIMER_CALLBACK: + RestartTimerCallback(event); + break; + } + + delete event; + return true; +} diff --git a/src/bgp/bgp_peer_close.h b/src/bgp/bgp_peer_close.h index e7d1d84a838..07de7cbcdfc 100644 --- a/src/bgp/bgp_peer_close.h +++ b/src/bgp/bgp_peer_close.h @@ -35,6 +35,39 @@ class BgpTable; // class PeerCloseManager { public: + PeerCloseManager(IPeerClose *peer_close, + boost::asio::io_service &io_service); + explicit PeerCloseManager(IPeerClose *peer_close); + virtual ~PeerCloseManager(); + + bool IsMembershipInUse() const { + return membership_state_ == MEMBERSHIP_IN_USE; + } + bool IsMembershipInWait() const { + return membership_state_ == MEMBERSHIP_IN_WAIT; + } + IPeerClose *peer_close() const { return peer_close_; } + IPeerClose::Families *families() { return &families_; } + + bool IsCloseInProgress() const { return state_ != NONE; } + bool IsInGracefulRestartTimerWait() const { + return state_ == GR_TIMER || state_ == LLGR_TIMER; + } + bool IsQueueEmpty() const { return event_queue_->IsQueueEmpty(); } + + void Close(bool non_graceful); + void ProcessEORMarkerReceived(Address::Family family); + void MembershipRequest(); + virtual void MembershipRequestCallback(); + void FillCloseInfo(BgpNeighborResp *resp) const; + bool MembershipPathCallback(DBTablePartBase *root, BgpRoute *rt, + BgpPath *path); + +private: + friend class PeerCloseTest; + friend class PeerCloseManagerTest; + friend class GracefulRestartTest; + enum State { BEGIN_STATE, NONE = BEGIN_STATE, @@ -53,38 +86,34 @@ class PeerCloseManager { MEMBERSHIP_IN_WAIT }; - explicit PeerCloseManager(IPeerClose *peer_close, - boost::asio::io_service &io_service); - explicit PeerCloseManager(IPeerClose *peer_close); - virtual ~PeerCloseManager(); - - MembershipState membership_state() const { return membership_state_; } - void set_membership_state(MembershipState state) { - membership_state_ = state; - } - - bool IsCloseInProgress() const { - tbb::mutex::scoped_lock lock(mutex_); - return state_ != NONE; - } - - bool IsInGracefulRestartTimerWait() const { - tbb::mutex::scoped_lock lock(mutex_); - return state_ == GR_TIMER || state_ == LLGR_TIMER; - } - - State state() const { - tbb::mutex::scoped_lock lock(mutex_); - return state_; - } - - void set_state(State state) { state_ = state; } - void Close(bool non_graceful); - void ProcessEORMarkerReceived(Address::Family family); - void MembershipRequest(); + enum EventType { + BEGIN_EVENT, + EVENT_NONE = BEGIN_EVENT, + CLOSE, + EOR_RECEIVED, + MEMBERSHIP_REQUEST, + MEMBERSHIP_REQUEST_COMPLETE_CALLBACK, + TIMER_CALLBACK, + END_EVENT = TIMER_CALLBACK + }; - bool RestartTimerCallback(); - void FillCloseInfo(BgpNeighborResp *resp) const; + struct Event { + Event(EventType event_type, bool non_graceful) : + event_type(event_type), non_graceful(non_graceful), + family(Address::UNSPEC) { } + Event(EventType event_type, Address::Family family) : + event_type(event_type), non_graceful(false), + family(family) { } + Event(EventType event_type) : event_type(event_type), + non_graceful(false), + family(Address::UNSPEC) { } + Event() : event_type(EVENT_NONE), non_graceful(false), + family(Address::UNSPEC) { } + + EventType event_type; + bool non_graceful; + Address::Family family; + }; struct Stats { Stats() { memset(this, 0, sizeof(Stats)); } @@ -99,43 +128,38 @@ class PeerCloseManager { uint64_t gr_timer; uint64_t llgr_timer; }; - const Stats &stats() const { return stats_; } - bool MembershipRequestCallback(); - bool MembershipPathCallback(DBTablePartBase *root, BgpRoute *rt, - BgpPath *path); - IPeerClose *peer_close() const { return peer_close_; } - bool close_again() const { return close_again_; } - IPeerClose::Families *families() { return &families_; } - void set_membership_req_pending(int count) { - membership_req_pending_ = count; - } - -protected: - tbb::atomic membership_req_pending_; - -private: - friend class PeerCloseManagerTest; + State state() const { return state_; } + const Stats &stats() const { return stats_; } + void Close(Event *event); + void ProcessEORMarkerReceived(Event *event); virtual void StartRestartTimer(int time); + bool RestartTimerCallback(); + void RestartTimerCallback(Event *event); void ProcessClosure(); void CloseComplete(); - bool ProcessSweepStateActions(); - virtual void TriggerSweepStateActions(); + void TriggerSweepStateActions(); std::string GetStateName(State state) const; std::string GetMembershipStateName(MembershipState state) const; void CloseInternal(); - bool MembershipRequestCompleteCallbackInternal(); - void MembershipRequestInternal(); + void MembershipRequest(Event *event); virtual bool CanUseMembershipManager() const; virtual BgpMembershipManager *membership_mgr() const; - virtual bool GRTimerFired() const; - virtual void StaleNotify(); - bool NotifyStaleEvent(); + bool MembershipRequestCallback(Event *event); + void StaleNotify(); + bool EventCallback(Event *event); + void EnqueueEvent(Event *event) { event_queue_->Enqueue(event); } + bool close_again() const { return close_again_; } + void set_membership_req_pending(int count) { + membership_req_pending_ = count; + } + void set_membership_state(MembershipState state) { + membership_state_ = state; + } IPeerClose *peer_close_; Timer *stale_timer_; - Timer *sweep_timer_; - Timer *stale_notify_timer_; + boost::scoped_ptr > event_queue_; State state_; bool close_again_; bool non_graceful_; @@ -144,7 +168,7 @@ class PeerCloseManager { MembershipState membership_state_; IPeerClose::Families families_; Stats stats_; - mutable tbb::mutex mutex_; + tbb::atomic membership_req_pending_; }; #endif // SRC_BGP_BGP_PEER_CLOSE_H_ diff --git a/src/bgp/bgp_xmpp_channel.cc b/src/bgp/bgp_xmpp_channel.cc index e707410dcea..1bbeea6fe84 100644 --- a/src/bgp/bgp_xmpp_channel.cc +++ b/src/bgp/bgp_xmpp_channel.cc @@ -207,6 +207,15 @@ class BgpXmppChannel::PeerClose : public IPeerClose { parent_->ReceiveEndOfRIB(family); } + // Process any pending subscriptions if close manager is now no longer + // using membership manager. + virtual void MembershipRequestCallbackComplete(bool result) { + if (result && parent_) { + assert(parent_->membership_in_use()); + parent_->ProcessPendingSubscriptions(); + } + } + virtual void CustomClose() { if (!parent_ || parent_->rtarget_routes_.empty()) return; @@ -409,13 +418,11 @@ class BgpXmppChannel::XmppPeer : public IPeer { virtual bool MembershipPathCallback(DBTablePartBase *tpart, BgpRoute *rt, BgpPath *path) { PeerCloseManager *close_manager = peer_close()->close_manager(); - if (close_manager->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_USE) { + if (close_manager->IsMembershipInUse()) return close_manager->MembershipPathCallback(tpart, rt, path); - } else { - BgpTable *table = static_cast(tpart->parent()); - return table->DeletePath(tpart, rt, path); - } + + BgpTable *table = static_cast(tpart->parent()); + return table->DeletePath(tpart, rt, path); } virtual bool SendUpdate(const uint8_t *msg, size_t msgsize); @@ -586,6 +593,7 @@ BgpXmppChannel::BgpXmppChannel(XmppChannel *channel, BgpServer *bgp_server, delete_in_progress_(false), deleted_(false), defer_peer_close_(false), + membership_in_use_(false), end_of_rib_timer_(NULL), membership_response_worker_( TaskScheduler::GetInstance()->GetTaskId("xmpp::StateMachine"), @@ -616,8 +624,7 @@ BgpXmppChannel::~BgpXmppChannel() { manager_->decrement_deleting_count(); STLDeleteElements(&defer_q_); assert(peer_deleted()); - assert(peer_->peer_close()->close_manager()->membership_state() != - PeerCloseManager::MEMBERSHIP_IN_USE); + assert(!peer_->peer_close()->close_manager()->IsMembershipInUse()); assert(routingtable_membership_request_map_.empty()); TimerManager::DeleteTimer(end_of_rib_timer_); BGP_LOG_PEER(Event, peer_.get(), SandeshLevel::SYS_INFO, BGP_LOG_FLAG_ALL, @@ -1898,8 +1905,7 @@ bool BgpXmppChannel::ResumeClose() { void BgpXmppChannel::RegisterTable(int line, BgpTable *table, int instance_id) { // Defer if Membership manager is in use (by close manager). - if (peer_close_->close_manager()->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_USE) { + if (membership_in_use_) { BGP_LOG_PEER_TABLE(Peer(), SandeshLevel::SYS_DEBUG, BGP_LOG_FLAG_ALL, table, "RegisterTable deferred " "from :" << line); @@ -1916,9 +1922,9 @@ void BgpXmppChannel::RegisterTable(int line, BgpTable *table, int instance_id) { } void BgpXmppChannel::UnregisterTable(int line, BgpTable *table) { + // Defer if Membership manager is in use (by close manager). - if (peer_close_->close_manager()->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_USE) { + if (membership_in_use_) { BGP_LOG_PEER_TABLE(Peer(), SandeshLevel::SYS_DEBUG, BGP_LOG_FLAG_ALL, table, "UnregisterTable deferred " "from :" << line); @@ -1938,8 +1944,8 @@ void BgpXmppChannel::UnregisterTable(int line, BgpTable *table) { // Process all pending membership requests of various tables. void BgpXmppChannel::ProcessPendingSubscriptions() { - assert(peer_close_->close_manager()->membership_state() != - PeerCloseManager::MEMBERSHIP_IN_USE); + membership_in_use_ = false; + assert(!peer_close_->close_manager()->IsMembershipInUse()); BOOST_FOREACH(RoutingTableMembershipRequestMap::value_type &i, routingtable_membership_request_map_) { BgpTable *table = static_cast( @@ -1955,10 +1961,9 @@ void BgpXmppChannel::ProcessPendingSubscriptions() { } bool BgpXmppChannel::MembershipResponseHandler(string table_name) { - if (peer_close_->close_manager()->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_USE) { - if (Peer()->peer_close()->close_manager()->MembershipRequestCallback()) - ProcessPendingSubscriptions(); + if (peer_close_->close_manager()->IsMembershipInUse()) { + membership_in_use_ = true; + Peer()->peer_close()->close_manager()->MembershipRequestCallback(); return true; } @@ -1996,8 +2001,7 @@ bool BgpXmppChannel::MembershipResponseHandler(string table_name) { } // If Close manager is waiting to use membership, try now. - if (peer_close_->close_manager()->membership_state() == - PeerCloseManager::MEMBERSHIP_IN_WAIT) + if (peer_close_->close_manager()->IsMembershipInWait()) peer_close_->close_manager()->MembershipRequest(); return true; diff --git a/src/bgp/bgp_xmpp_channel.h b/src/bgp/bgp_xmpp_channel.h index 1c772126efb..d6b54d42878 100644 --- a/src/bgp/bgp_xmpp_channel.h +++ b/src/bgp/bgp_xmpp_channel.h @@ -99,6 +99,7 @@ class BgpXmppChannel { void set_peer_closed(bool flag); bool peer_deleted() const; + bool membership_in_use() const { return membership_in_use_; } uint64_t peer_closed_at() const; bool routingtable_membership_request_map_empty() const; size_t GetMembershipRequestQueueSize() const; @@ -268,6 +269,7 @@ class BgpXmppChannel { bool delete_in_progress_; bool deleted_; bool defer_peer_close_; + bool membership_in_use_; Timer *end_of_rib_timer_; WorkQueue membership_response_worker_; SubscribedRoutingInstanceList routing_instances_; diff --git a/src/bgp/ipeer.h b/src/bgp/ipeer.h index 396eda1ad9d..70f166de049 100644 --- a/src/bgp/ipeer.h +++ b/src/bgp/ipeer.h @@ -170,6 +170,7 @@ class IPeerClose { virtual bool IsReady() const = 0; virtual IPeer *peer() const = 0; virtual void ReceiveEndOfRIB(Address::Family family) = 0; + virtual void MembershipRequestCallbackComplete(bool result) = 0; }; class IPeer : public IPeerUpdate { diff --git a/src/bgp/test/graceful_restart_test.cc b/src/bgp/test/graceful_restart_test.cc index 1012df9778c..ee49d460cee 100644 --- a/src/bgp/test/graceful_restart_test.cc +++ b/src/bgp/test/graceful_restart_test.cc @@ -190,6 +190,13 @@ static void process_command_line_args(int argc, char **argv) { n_peers.push_back(npeers); } + // Retrieve logging params. + if (vm.count("log-category")) + d_log_category_ = vm["log-category"].as(); + + if (vm.count("log-level")) + d_log_level_ = vm["log-level"].as(); + if (d_log_disable_) { SetLoggingDisabled(true); } @@ -248,18 +255,15 @@ static vector GetTargetParameters() { class PeerCloseManagerTest : public PeerCloseManager { public: explicit PeerCloseManagerTest(IPeerClose *peer_close) : - PeerCloseManager(peer_close), gr_timer_fired_(false) { + PeerCloseManager(peer_close) { } ~PeerCloseManagerTest() { last_stats_ = stats(); } static Stats &last_stats() { return last_stats_; } static void reset_last_stats() { memset(&last_stats_, 0, sizeof(PeerCloseManagerTest::last_stats())); } - virtual bool GRTimerFired() const { return gr_timer_fired_; } - void set_gr_timer_fired(bool flag) { gr_timer_fired_ = flag; } private: - bool gr_timer_fired_; static Stats last_stats_; }; @@ -1078,13 +1082,10 @@ bool GracefulRestartTest::SkipNotificationReceive(BgpPeerTest *peer, // Invoke stale timer callbacks directly to speed up. void GracefulRestartTest::GRTimerCallback(PeerCloseManagerTest *pc) { - CHECK_CONCURRENCY("bgp::Config"); + CHECK_CONCURRENCY("timer::TimerTask"); // Fire the timer. - pc->set_gr_timer_fired(true); - if (pc->RestartTimerCallback()) - assert(!pc->RestartTimerCallback()); - pc->set_gr_timer_fired(false); + assert(!pc->RestartTimerCallback()); } void GracefulRestartTest::FireGRTimer(PeerCloseManagerTest *pc, bool is_ready) { @@ -1093,7 +1094,7 @@ void GracefulRestartTest::FireGRTimer(PeerCloseManagerTest *pc, bool is_ready) { if (is_ready) { uint64_t sweep = pc->stats().sweep; TaskFire(boost::bind(&GracefulRestartTest::GRTimerCallback, this, pc), - "bgp::Config"); + "timer::TimerTask"); TASK_UTIL_EXPECT_EQ(sweep + 1, pc->stats().sweep); TASK_UTIL_EXPECT_EQ(PeerCloseManager::NONE, pc->state()); WaitForIdle(); @@ -1109,7 +1110,7 @@ void GracefulRestartTest::FireGRTimer(PeerCloseManagerTest *pc, bool is_ready) { if (pc->state() == PeerCloseManager::GR_TIMER || pc->state() == PeerCloseManager::LLGR_TIMER) TaskFire(boost::bind(&GracefulRestartTest::GRTimerCallback, - this, pc), "bgp::Config"); + this, pc), "timer::TimerTask"); WaitForIdle(); stats = is_xmpp ? PeerCloseManagerTest::last_stats() : pc->stats(); if (stats.deletes > deletes) diff --git a/src/bgp/test/peer_close_test.cc b/src/bgp/test/peer_close_test.cc index 2bcffe15ab6..6169ade0fdd 100644 --- a/src/bgp/test/peer_close_test.cc +++ b/src/bgp/test/peer_close_test.cc @@ -9,6 +9,7 @@ #include "bgp/bgp_log.h" #include "bgp/bgp_membership.h" #include "bgp/bgp_peer_close.h" +#include "io/test/event_manager_test.h" class PeerCloseManagerTest; @@ -16,7 +17,8 @@ class IPeerCloseTest : public IPeerClose { public: IPeerCloseTest() : graceful_(false), ll_graceful_(false), is_ready_(false), close_graceful_(false), deleted_(false), swept_(false), - stale_(false), llgr_stale_(false) { + stale_(false), llgr_stale_(false), + membership_request_complete_result_(false) { } virtual ~IPeerCloseTest() { } @@ -36,6 +38,9 @@ class IPeerCloseTest : public IPeerClose { virtual void GetGracefulRestartFamilies(Families *families) const { families->insert(Address::INET); } + virtual void MembershipRequestCallbackComplete(bool result) { + membership_request_complete_result_ = result; + } virtual const int GetGracefulRestartTime() const { return 123; } virtual const int GetLongLivedGracefulRestartTime() const { return 321; } virtual bool IsReady() const { return is_ready_; } @@ -59,6 +64,9 @@ class IPeerCloseTest : public IPeerClose { bool swept() const { return swept_; } bool stale() const { return stale_; } bool llgr_stale() const { return llgr_stale_; } + bool membership_request_complete_result() const { + return membership_request_complete_result_; + } private: PeerCloseManagerTest *close_manager_; @@ -70,27 +78,18 @@ class IPeerCloseTest : public IPeerClose { bool swept_; bool stale_; bool llgr_stale_; + bool membership_request_complete_result_; }; class PeerCloseManagerTest : public PeerCloseManager { public: - enum Event { - BEGIN_EVENT, - CLOSE = BEGIN_EVENT, - EOR_RECEIVED, - MEMBERSHIP_REQUEST_COMPLETE_CALLBACK, - TIMER_CALLBACK, - END_EVENT = TIMER_CALLBACK - }; - PeerCloseManagerTest(IPeerClose *peer_close, boost::asio::io_service &io_service) : PeerCloseManager(peer_close, io_service), restart_time_(0), - restart_timer_started_(false), restart_timer_fired_(false) { + restart_timer_started_(false) { } ~PeerCloseManagerTest() { } Timer *stale_timer() const { return stale_timer_; } - Timer *sweep_timer() const { return sweep_timer_; } int GetMembershipRequestCount() const { return membership_req_pending_; } @@ -99,336 +98,381 @@ class PeerCloseManagerTest : public PeerCloseManager { virtual void StartRestartTimer(int time) { restart_time_ = time; restart_timer_started_ = true; + PeerCloseManager::StartRestartTimer(time); } - virtual void TriggerSweepStateActions() { - ConcurrencyScope scope("bgp::Config"); - ProcessSweepStateActions(); - } - virtual void StaleNotify() { - ConcurrencyScope scope("bgp::Config"); - PeerCloseManager::NotifyStaleEvent(); - } - uint32_t restart_time() const { return restart_time_; } + int restart_time() const { return restart_time_; } bool restart_timer_started() const { return restart_timer_started_; } - bool GRTimerFired() const { return restart_timer_fired_; } - void set_restart_timer_fired(bool flag) { restart_timer_fired_ = flag; } - -private: - friend class PeerCloseTest; - uint32_t restart_time_; - bool restart_timer_started_; - bool restart_timer_fired_; -}; - -void IPeerCloseTest::ReceiveEndOfRIB(Address::Family family) { - close_manager_->ProcessEORMarkerReceived(family); -} - -typedef std::tr1::tuple TestParams; -class PeerCloseTest : public ::testing::TestWithParam { -public: - virtual void SetUp() { - peer_close_.reset(new IPeerCloseTest()); - close_manager_.reset(new PeerCloseManagerTest(peer_close_.get(), - io_service_)); - peer_close_->set_close_manager(close_manager_.get()); - - state_ = static_cast( - std::tr1::get<0>(GetParam())); - event_ = static_cast( - std::tr1::get<1>(GetParam())); - peer_close_->set_is_ready(std::tr1::get<2>(GetParam())); - peer_close_->set_graceful(std::tr1::get<3>(GetParam())); - peer_close_->set_ll_graceful(std::tr1::get<4>(GetParam())); - peer_close_->set_close_graceful(std::tr1::get<5>(GetParam())); - pending_requests_ = std::tr1::get<6>(GetParam()); - membership_request_complete_result_ = false; + void Run(); + static int GetBeginState() { return BEGIN_STATE; } + static int GetEndState() { return END_STATE; } + static int GetBeginEvent() { return BEGIN_EVENT; } + static int GetEndEvent() { return END_EVENT; } + + void SetUp(int state, int event, int pending_requests, + IPeerCloseTest *peer_close) { + test_state_ = static_cast(state); + test_event_ = static_cast(event); + pending_requests_ = pending_requests; + peer_close_ = peer_close; } - virtual void TearDown() { } void GotoState() { - close_manager_->set_state(state_); - switch (state_) { - case PeerCloseManager::NONE: + state_ = test_state_; + switch (test_state_) { + case NONE: break; - case PeerCloseManager::GR_TIMER: - peer_close_->GetGracefulRestartFamilies( - close_manager_->families()); + case GR_TIMER: + peer_close_->GetGracefulRestartFamilies(families()); break; - case PeerCloseManager::LLGR_TIMER: - peer_close_->GetGracefulRestartFamilies( - close_manager_->families()); + case LLGR_TIMER: + peer_close_->GetGracefulRestartFamilies(families()); break; - case PeerCloseManager::SWEEP: + case SWEEP: break; - case PeerCloseManager::DELETE: + case DELETE: break; - case PeerCloseManager::STALE: + case STALE: break; - case PeerCloseManager::LLGR_STALE: + case LLGR_STALE: break; } } + virtual void MembershipRequestCallback() { + PeerCloseManager::MembershipRequestCallback(); + task_util::WaitForIdle(); + } + void TriggerEvent() { - switch (event_) { - case PeerCloseManagerTest::CLOSE: - close_manager_->Close(!peer_close_->close_graceful()); - break; - case PeerCloseManagerTest::EOR_RECEIVED: - close_manager_->ProcessEORMarkerReceived(Address::INET); - break; - case PeerCloseManagerTest::MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: - close_manager_->set_membership_state( - PeerCloseManager::MEMBERSHIP_IN_USE); - close_manager_->set_membership_req_pending(pending_requests_); - if (!(close_manager_->state() == PeerCloseManager::STALE || - close_manager_->state() == PeerCloseManager::LLGR_STALE || - close_manager_->state() == PeerCloseManager::SWEEP || - close_manager_->state() == PeerCloseManager::DELETE)) { - TASK_UTIL_EXPECT_DEATH( - close_manager_->MembershipRequestCallback(), ".*"); + switch (test_event_) { + case EVENT_NONE: + break; + case CLOSE: + Close(!peer_close_->close_graceful()); + break; + case EOR_RECEIVED: + ProcessEORMarkerReceived(Address::INET); + break; + case MEMBERSHIP_REQUEST: + break; + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + set_membership_state(MEMBERSHIP_IN_USE); + set_membership_req_pending(pending_requests_); + if (!(state() == STALE || state() == LLGR_STALE || + state() == SWEEP || state() == DELETE)) { + TASK_UTIL_EXPECT_DEATH(MembershipRequestCallback(), ".*"); } else { - membership_request_complete_result_ = - close_manager_->MembershipRequestCallback(); + MembershipRequestCallback(); } break; - case PeerCloseManagerTest::TIMER_CALLBACK: - ConcurrencyScope scope("bgp::Config"); - close_manager_->set_restart_timer_fired(true); - if (close_manager_->RestartTimerCallback()) - close_manager_->RestartTimerCallback(); - close_manager_->set_restart_timer_fired(false); + case TIMER_CALLBACK: + task_util::TaskFire( + boost::bind(&PeerCloseManager::RestartTimerCallback, this), + "timer::TimerTask"); + task_util::WaitForIdle(); break; } } void check_close_at_start() { if (!peer_close_->graceful() || !peer_close_->close_graceful()) { - EXPECT_EQ(PeerCloseManager::DELETE, close_manager_->state()); - EXPECT_FALSE(close_manager_->stale_timer()->running()); - EXPECT_FALSE(close_manager_->sweep_timer()->running()); + TASK_UTIL_EXPECT_EQ(DELETE, state()); + TASK_UTIL_EXPECT_FALSE(stale_timer()->running()); // Expect Membership (walk) request. - EXPECT_TRUE(close_manager_->GetMembershipRequestCount()); + TASK_UTIL_EXPECT_TRUE(GetMembershipRequestCount()); // Trigger unregister peer rib walks completion. - close_manager_->MembershipRequestCallback(); - EXPECT_EQ(PeerCloseManager::NONE, close_manager_->state()); + MembershipRequestCallback(); + TASK_UTIL_EXPECT_EQ(NONE, state()); } else { // Peer supports GR and Graceful closure has been triggered. - EXPECT_EQ(PeerCloseManager::STALE, close_manager_->state()); - EXPECT_TRUE(peer_close_->stale()); - EXPECT_FALSE(close_manager_->stale_timer()->running()); + TASK_UTIL_EXPECT_EQ(STALE, state()); + TASK_UTIL_EXPECT_TRUE(peer_close_->stale()); + TASK_UTIL_EXPECT_FALSE(stale_timer()->running()); } } -protected: - boost::scoped_ptr close_manager_; - boost::scoped_ptr peer_close_; - boost::asio::io_service io_service_; - PeerCloseManagerTest::State state_; - PeerCloseManagerTest::Event event_; +private: + friend class PeerCloseTest; + int restart_time_; + bool restart_timer_started_; + State test_state_; + EventType test_event_; int pending_requests_; - bool membership_request_complete_result_; + IPeerCloseTest *peer_close_; }; +void IPeerCloseTest::ReceiveEndOfRIB(Address::Family family) { + close_manager_->ProcessEORMarkerReceived(family); +} PeerCloseManager *IPeerCloseTest::close_manager() { return close_manager_; } -// Non graceful closure. Expect membership walk for ribin and ribout deletes. -TEST_P(PeerCloseTest, Test) { - GotoState(); - TriggerEvent(); - - switch (state_) { - case PeerCloseManagerTest::NONE: - switch (event_) { - case PeerCloseManagerTest::CLOSE: +void PeerCloseManagerTest::Run() { + switch (test_state_) { + case NONE: + switch (test_event_) { + case EVENT_NONE: + break; + case MEMBERSHIP_REQUEST: + break; + case CLOSE: check_close_at_start(); break; - case PeerCloseManagerTest::EOR_RECEIVED: - EXPECT_FALSE(close_manager_->stale_timer()->running()); - EXPECT_FALSE(close_manager_->sweep_timer()->running()); + case EOR_RECEIVED: + TASK_UTIL_EXPECT_FALSE(stale_timer()->running()); // No change in state for eor reception in normal lifetime of peer. - EXPECT_EQ(PeerCloseManager::NONE, close_manager_->state()); + TASK_UTIL_EXPECT_EQ(NONE, state()); break; - case PeerCloseManagerTest::MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: break; - case PeerCloseManagerTest::TIMER_CALLBACK: - EXPECT_EQ(PeerCloseManager::NONE, close_manager_->state()); + case TIMER_CALLBACK: + TASK_UTIL_EXPECT_EQ(NONE, state()); break; } break; - case PeerCloseManagerTest::STALE: - switch (event_) { - case PeerCloseManagerTest::CLOSE: - EXPECT_TRUE(close_manager_->close_again()); + case STALE: + switch (test_event_) { + case EVENT_NONE: + break; + case MEMBERSHIP_REQUEST: break; - case PeerCloseManagerTest::EOR_RECEIVED: - EXPECT_FALSE(close_manager_->stale_timer()->running()); - EXPECT_FALSE(close_manager_->sweep_timer()->running()); + case CLOSE: + TASK_UTIL_EXPECT_TRUE(close_again()); + break; + case EOR_RECEIVED: + TASK_UTIL_EXPECT_FALSE(stale_timer()->running()); // No change in state for eor reception in normal lifetime of peer. - EXPECT_EQ(PeerCloseManager::STALE, close_manager_->state()); + TASK_UTIL_EXPECT_EQ(STALE, state()); break; - case PeerCloseManagerTest::MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: if (pending_requests_ > 1) { - EXPECT_FALSE(membership_request_complete_result_); + TASK_UTIL_EXPECT_FALSE( + peer_close_->membership_request_complete_result()); break; } - EXPECT_TRUE(membership_request_complete_result_); - EXPECT_EQ(PeerCloseManager::GR_TIMER, close_manager_->state()); - EXPECT_TRUE(close_manager_->restart_timer_started()); - EXPECT_EQ(1000 * peer_close_->GetGracefulRestartTime(), - close_manager_->restart_time()); + TASK_UTIL_EXPECT_TRUE( + peer_close_->membership_request_complete_result()); + TASK_UTIL_EXPECT_EQ(GR_TIMER, state()); + TASK_UTIL_EXPECT_TRUE(restart_timer_started()); + TASK_UTIL_EXPECT_EQ(1000 * peer_close_->GetGracefulRestartTime(), + restart_time()); break; - case PeerCloseManagerTest::TIMER_CALLBACK: - EXPECT_EQ(PeerCloseManager::STALE, close_manager_->state()); + case TIMER_CALLBACK: + TASK_UTIL_EXPECT_EQ(STALE, state()); break; } break; - case PeerCloseManagerTest::GR_TIMER: - switch (event_) { - case PeerCloseManagerTest::CLOSE: + case GR_TIMER: + switch (test_event_) { + case EVENT_NONE: + break; + case MEMBERSHIP_REQUEST: + break; + case CLOSE: // Nested closures trigger GR restart check_close_at_start(); break; - case PeerCloseManagerTest::EOR_RECEIVED: + case EOR_RECEIVED: // Expect GR timer to start right away to exit from GR. - EXPECT_TRUE(close_manager_->restart_timer_started()); - EXPECT_EQ(0, close_manager_->restart_time()); + TASK_UTIL_EXPECT_TRUE(restart_timer_started()); + TASK_UTIL_EXPECT_EQ(0, restart_time()); break; - case PeerCloseManagerTest::MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: break; - case PeerCloseManagerTest::TIMER_CALLBACK: + case TIMER_CALLBACK: if (peer_close_->IsReady()) { - EXPECT_EQ(PeerCloseManager::SWEEP, close_manager_->state()); + TASK_UTIL_EXPECT_EQ(SWEEP, state()); break; } if (peer_close_->IsCloseLongLivedGraceful()) { - EXPECT_EQ(PeerCloseManager::LLGR_STALE, - close_manager_->state()); - EXPECT_TRUE(peer_close_->llgr_stale()); + TASK_UTIL_EXPECT_EQ(LLGR_STALE, + state()); + TASK_UTIL_EXPECT_TRUE(peer_close_->llgr_stale()); break; } - EXPECT_EQ(PeerCloseManager::DELETE, close_manager_->state()); + TASK_UTIL_EXPECT_EQ(DELETE, state()); break; } break; - case PeerCloseManagerTest::LLGR_STALE: - switch (event_) { - case PeerCloseManagerTest::CLOSE: - EXPECT_TRUE(close_manager_->close_again()); + case LLGR_STALE: + switch (test_event_) { + case EVENT_NONE: + break; + case MEMBERSHIP_REQUEST: + break; + case CLOSE: + TASK_UTIL_EXPECT_TRUE(close_again()); break; - case PeerCloseManagerTest::EOR_RECEIVED: - EXPECT_FALSE(close_manager_->stale_timer()->running()); - EXPECT_FALSE(close_manager_->sweep_timer()->running()); + case EOR_RECEIVED: + TASK_UTIL_EXPECT_FALSE(stale_timer()->running()); break; - case PeerCloseManagerTest::MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: if (pending_requests_ > 1) { - EXPECT_FALSE(membership_request_complete_result_); + TASK_UTIL_EXPECT_FALSE( + peer_close_->membership_request_complete_result()); break; } - EXPECT_TRUE(membership_request_complete_result_); - EXPECT_EQ(PeerCloseManager::LLGR_TIMER, close_manager_->state()); - EXPECT_TRUE(close_manager_->restart_timer_started()); - EXPECT_EQ(1000 * peer_close_->GetLongLivedGracefulRestartTime(), - close_manager_->restart_time()); + TASK_UTIL_EXPECT_TRUE( + peer_close_->membership_request_complete_result()); + TASK_UTIL_EXPECT_EQ(LLGR_TIMER, state()); + TASK_UTIL_EXPECT_TRUE(restart_timer_started()); + TASK_UTIL_EXPECT_EQ( + 1000 * peer_close_->GetLongLivedGracefulRestartTime(), + restart_time()); break; - case PeerCloseManagerTest::TIMER_CALLBACK: - EXPECT_EQ(PeerCloseManager::LLGR_STALE, close_manager_->state()); + case TIMER_CALLBACK: + TASK_UTIL_EXPECT_EQ(LLGR_STALE, state()); break; } break; - case PeerCloseManagerTest::LLGR_TIMER: - switch (event_) { - case PeerCloseManagerTest::CLOSE: + case LLGR_TIMER: + switch (test_event_) { + case EVENT_NONE: + break; + case MEMBERSHIP_REQUEST: + break; + case CLOSE: // Nested closures trigger GR restart check_close_at_start(); break; - case PeerCloseManagerTest::EOR_RECEIVED: + case EOR_RECEIVED: // Expect GR timer to start right away to exit from GR. - EXPECT_TRUE(close_manager_->restart_timer_started()); - EXPECT_EQ(0, close_manager_->restart_time()); + TASK_UTIL_EXPECT_TRUE(restart_timer_started()); + TASK_UTIL_EXPECT_EQ(0, restart_time()); break; - case PeerCloseManagerTest::MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: break; - case PeerCloseManagerTest::TIMER_CALLBACK: + case TIMER_CALLBACK: if (peer_close_->IsReady()) { - EXPECT_EQ(PeerCloseManager::SWEEP, close_manager_->state()); + TASK_UTIL_EXPECT_EQ(SWEEP, state()); break; } if (peer_close_->IsCloseLongLivedGraceful()) { - EXPECT_EQ(PeerCloseManager::DELETE, close_manager_->state()); + TASK_UTIL_EXPECT_EQ(DELETE, state()); break; } - EXPECT_EQ(PeerCloseManager::DELETE, close_manager_->state()); + TASK_UTIL_EXPECT_EQ(DELETE, state()); break; } break; - case PeerCloseManagerTest::SWEEP: - switch (event_) { - case PeerCloseManagerTest::CLOSE: - EXPECT_TRUE(close_manager_->close_again()); + case SWEEP: + switch (test_event_) { + case EVENT_NONE: + break; + case MEMBERSHIP_REQUEST: break; - case PeerCloseManagerTest::EOR_RECEIVED: - EXPECT_EQ(PeerCloseManager::SWEEP, close_manager_->state()); + case CLOSE: + TASK_UTIL_EXPECT_TRUE(close_again()); break; - case PeerCloseManagerTest::MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + case EOR_RECEIVED: + TASK_UTIL_EXPECT_EQ(SWEEP, state()); + break; + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: if (pending_requests_ > 1) { - EXPECT_FALSE(membership_request_complete_result_); + TASK_UTIL_EXPECT_FALSE( + peer_close_->membership_request_complete_result()); break; } - EXPECT_TRUE(membership_request_complete_result_); - EXPECT_EQ(PeerCloseManager::NONE, close_manager_->state()); - EXPECT_TRUE(peer_close_->swept()); + TASK_UTIL_EXPECT_TRUE( + peer_close_->membership_request_complete_result()); + TASK_UTIL_EXPECT_EQ(NONE, state()); + TASK_UTIL_EXPECT_TRUE(peer_close_->swept()); break; - case PeerCloseManagerTest::TIMER_CALLBACK: - EXPECT_EQ(PeerCloseManager::SWEEP, close_manager_->state()); + case TIMER_CALLBACK: + TASK_UTIL_EXPECT_EQ(SWEEP, state()); break; } break; - case PeerCloseManagerTest::DELETE: - switch (event_) { - case PeerCloseManagerTest::CLOSE: - EXPECT_TRUE(close_manager_->close_again()); + case DELETE: + switch (test_event_) { + case EVENT_NONE: + break; + case MEMBERSHIP_REQUEST: + break; + case CLOSE: + TASK_UTIL_EXPECT_TRUE(close_again()); break; - case PeerCloseManagerTest::EOR_RECEIVED: - EXPECT_EQ(PeerCloseManager::DELETE, close_manager_->state()); + case EOR_RECEIVED: + TASK_UTIL_EXPECT_EQ(DELETE, state()); break; - case PeerCloseManagerTest::MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: + case MEMBERSHIP_REQUEST_COMPLETE_CALLBACK: if (pending_requests_ > 1) { - EXPECT_FALSE(membership_request_complete_result_); + TASK_UTIL_EXPECT_FALSE( + peer_close_->membership_request_complete_result()); break; } - EXPECT_TRUE(membership_request_complete_result_); - EXPECT_EQ(PeerCloseManager::NONE, close_manager_->state()); - EXPECT_TRUE(peer_close_->deleted()); + TASK_UTIL_EXPECT_TRUE( + peer_close_->membership_request_complete_result()); + TASK_UTIL_EXPECT_EQ(NONE, state()); + TASK_UTIL_EXPECT_TRUE(peer_close_->deleted()); break; - case PeerCloseManagerTest::TIMER_CALLBACK: - EXPECT_EQ(PeerCloseManager::DELETE, close_manager_->state()); + case TIMER_CALLBACK: + TASK_UTIL_EXPECT_EQ(DELETE, state()); break; } break; } } +typedef std::tr1::tuple TestParams; +class PeerCloseTest : public ::testing::TestWithParam { +public: + PeerCloseTest() : thread_(&evm_) { } + + virtual void SetUp() { + peer_close_.reset(new IPeerCloseTest()); + close_manager_.reset(new PeerCloseManagerTest(peer_close_.get(), + *(evm_.io_service()))); + peer_close_->set_close_manager(close_manager_.get()); + close_manager_->SetUp(std::tr1::get<0>(GetParam()), + std::tr1::get<1>(GetParam()), + std::tr1::get<6>(GetParam()), peer_close_.get()); + peer_close_->set_is_ready(std::tr1::get<2>(GetParam())); + peer_close_->set_graceful(std::tr1::get<3>(GetParam())); + peer_close_->set_ll_graceful(std::tr1::get<4>(GetParam())); + peer_close_->set_close_graceful(std::tr1::get<5>(GetParam())); + + thread_.Start(); + } + + virtual void TearDown() { + evm_.Shutdown(); + thread_.Join(); + task_util::WaitForIdle(); + } + +protected: + boost::scoped_ptr close_manager_; + boost::scoped_ptr peer_close_; + EventManager evm_; + ServerThread thread_; +}; + +// Non graceful closure. Expect membership walk for ribin and ribout deletes. +TEST_P(PeerCloseTest, Test) { + close_manager_->GotoState(); + close_manager_->TriggerEvent(); + close_manager_->Run(); +} + INSTANTIATE_TEST_CASE_P(PeerCloseTestWithParams, PeerCloseTest, ::testing::Combine( - ::testing::Range(PeerCloseManagerTest::BEGIN_STATE, - PeerCloseManagerTest::END_STATE + 1), - ::testing::Range(PeerCloseManagerTest::BEGIN_EVENT, - PeerCloseManagerTest::END_EVENT + 1), + ::testing::Range(PeerCloseManagerTest::GetBeginState(), + PeerCloseManagerTest::GetEndState() + 1), + ::testing::Range(PeerCloseManagerTest::GetBeginEvent(), + PeerCloseManagerTest::GetEndEvent() + 1), ::testing::Bool(), ::testing::Bool(), ::testing::Bool(), ::testing::Bool(), ::testing::Range(1, 2)));