PostOffice是全局管理类，每个节点中都有且只有一个该类的对象， 用来配置当前node的一些信息，例如当前node是哪种类型(server,worker,scheduler)，分配nodeid以及worker/server 的rank 到 node id的转换。Postoffice类利用Singleton模式来保证只有一个对象。管理当前节点角色、其他节点的连接、心跳信息、配置信息。
节点分配：
InitEnvironment：根据环境变量设置创建Van，确定worker，server数量，并确定当前节点的角色；
1、调度节点
Scheduler节点管理所有节点的地址。每个节点要知道Scheduler节点的IP、port；启动时绑定一个本地端口，并向Scheduler节点报告。Scheduler节点在每个几点启动后，给节点分配ID，把节点信息通知出去（例如Worker节点要知道Server节点IP和端口，Server节点要知道Worker节点的IP和端口）。节点在建立连接后，才会正式启动。
/**
* Copyright (c) 2015 by Contributors
*/
#ifndef PS_INTERNAL_POSTOFFICE_H_
#define PS_INTERNAL_POSTOFFICE_H_
#include <mutex>
#include <algorithm>
#include <vector>
#include <unordered_map>
#include <memory>
#include "ps/range.h"
#include "ps/internal/env.h"
#include "ps/internal/customer.h"
#include "ps/internal/van.h"
namespace ps {
/**
* \brief the center of the system
*/
class Postoffice {
public:
/**
* \brief return the singleton object
*/
static Postoffice* Get() {
static Postoffice e; return &e;
}
/** \brief get the van */
Van* van() { return van_; }
/** 启动系统，函数将会阻塞直至所有节点启动 */
/**
* \brief start the system
*
* This function will block until every nodes are started.
* \param argv0 the program name, used for logging.
* \param do_barrier whether to block until every nodes are started.
*/
void Start(int customer_id, const char* argv0, const bool do_barrier);
/** 终止系统，在退出之前所有节点应该调用此函数，阻塞直至所有节点析构完成 */
/**
* \brief terminate the system
*
* All nodes should call this function before existing.
* \param do_barrier whether to do block until every node is finalized, default true.
*/
void Finalize(const int customer_id, const bool do_barrier = true);
/** 线程安全，增加customer至系统， */
/**
* \brief add an customer to the system. threadsafe
*/
void AddCustomer(Customer* customer);
/** 线程安全，通过给定ID移除customer */
/**
* \brief remove a customer by given it's id. threasafe
*/
void RemoveCustomer(Customer* customer);
/** 线程安全，通过ID获取系统的customer */
/**
* \brief get the customer by id, threadsafe
* \param app_id the application id
* \param customer_id the customer id
* \param timeout timeout in sec
* \return return nullptr if doesn't exist and timeout
*/
Customer* GetCustomer(int app_id, int customer_id, int timeout = 0) const;
/** 线程安全， 获取一个节点或组的ID */
/**
* \brief get the id of a node (group), threadsafe
*
* if it is a node group, return the list of node ids in this
* group. otherwise, return {node_id}
*/
const std::vector<int>& GetNodeIDs(int node_id) const {
const auto it = node_ids_.find(node_id);
CHECK(it != node_ids_.cend()) << "node " << node_id << " doesn't exist";
return it->second;
}
/** 获取所有server节点的key范围 */
/**
* \brief return the key ranges of all server nodes
*/
const std::vector<Range>& GetServerKeyRanges();
/**
* \brief the template of a callback
*/
using Callback = std::function<void()>;
/** 注册一个回调函数至系统，待终止系统时调用 */
/**
* \brief Register a callback to the system which is called after Finalize()
*
* The following codes are equal
* \code {cpp}
* RegisterExitCallback(cb);
* Finalize();
* \endcode
*
* \code {cpp}
* Finalize();
* cb();
* \endcode
* \param cb the callback function
*/
void RegisterExitCallback(const Callback& cb) {
exit_callback_ = cb;
}
/** 从worker的rank转换到节点ID */
/**
* \brief convert from a worker rank into a node id
* \param rank the worker rank
*/
static inline int WorkerRankToID(int rank) {
return rank * 2 + 9;
}
/** 从server的rank转换到节点ID */
/**
* \brief convert from a server rank into a node id
* \param rank the server rank
*/
static inline int ServerRankToID(int rank) {
return rank * 2 + 8;
}
/** 从server或worker的rank转换至节点ID */
/**
* \brief convert from a node id into a server or worker rank
* \param id the node id
*/
static inline int IDtoRank(int id) {
#ifdef _MSC_VER
#undef max
#endif
return std::max((id - 8) / 2, 0);
}
/** 返回worker节点数量 */
/** \brief Returns the number of worker nodes */
int num_workers() const { return num_workers_; }
/** 返回server节点数量 */
/** \brief Returns the number of server nodes */
int num_servers() const { return num_servers_; }
/**
* 返回节点在组中的rank，每个worker有一个唯一的rank，范围在0~Numworkers之间
* 对于server，类同；仅仅在Start调用之后，此函数可用；
*/
/** \brief Returns the rank of this node in its group
*
* Each worker will have a unique rank within [0, NumWorkers()). So are
* servers. This function is available only after \ref Start has been called.
*/
int my_rank() const { return IDtoRank(van_->my_node().id); }
/** 判别节点的类型：worker/server/scheduler */
/** \brief Returns true if this node is a worker node */
int is_worker() const { return is_worker_; }
/** \brief Returns true if this node is a server node. */
int is_server() const { return is_server_; }
/** \brief Returns true if this node is a scheduler node. */
int is_scheduler() const { return is_scheduler_; }
/** 返回verbose级别 */
/** \brief Returns the verbose level. */
int verbose() const { return verbose_; }
/** 返回节点是否是恢复的节点 */
/** \brief Return whether this node is a recovery node */
bool is_recovery() const { return van_->my_node().is_recovery; }
/** 阻碍 */
/**
* \brief barrier
* \param node_id the barrier group id
*/
void Barrier(int customer_id, int node_group);
/** 处理控制消息，由van调用 */
/**
* \brief process a control message, called by van
* \param the received message
*/
void Manage(const Message& recv);
/** 更新心跳记录表 */
/**
* \brief update the heartbeat record map
* \param node_id the \ref Node id
* \param t the last received heartbeat time
*/
void UpdateHeartbeat(int node_id, time_t t) {
std::lock_guard<std::mutex> lk(heartbeat_mu_);
heartbeats_[node_id] = t;
}
/** 获取超过t秒还未报告心跳的节点 */
/**
* \brief get node ids that haven't reported heartbeats for over t seconds
* \param t timeout in sec
*/
std::vector<int> GetDeadNodes(int t = 60);
private:
Postoffice();
~Postoffice() { delete van_; }
void InitEnvironment();
/** 通信功能类 */
Van* van_;
/** 互斥锁 */
mutable std::mutex mu_;
// app_id -> (customer_id -> customer pointer)
/** customer表 */
std::unordered_map<int, std::unordered_map<int, Customer*>> customers_;
/** 节点表 */
std::unordered_map<int, std::vector<int>> node_ids_;
/** server_key_ranges 锁 */
std::mutex server_key_ranges_mu_;
std::vector<Range> server_key_ranges_;
/** 区别角色 */
bool is_worker_, is_server_, is_scheduler_;
/** server/worker数量 */
int num_servers_, num_workers_;
std::unordered_map<int, std::unordered_map<int, bool> > barrier_done_;
int verbose_;
std::mutex barrier_mu_;
std::condition_variable barrier_cond_;
std::mutex heartbeat_mu_;
std::mutex start_mu_;
int init_stage_ = 0;
std::unordered_map<int, time_t> heartbeats_;
Callback exit_callback_;
/** \brief Holding a shared_ptr to prevent it from being destructed too early */
std::shared_ptr<Environment> env_ref_;
time_t start_time_;
DISALLOW_COPY_AND_ASSIGN(Postoffice);
};
/** \brief verbose log */
#define PS_VLOG(x) LOG_IF(INFO, x <= Postoffice::Get()->verbose())
} // namespace ps
#endif // PS_INTERNAL_POSTOFFICE_H_
