|
|
|
@ -117,7 +117,7 @@ typedef ngx_int_t (*ngx_http_upstream_check_packet_parse_pt)
|
|
|
|
|
typedef void (*ngx_http_upstream_check_packet_clean_pt)
|
|
|
|
|
(ngx_http_upstream_check_peer_t *peer);
|
|
|
|
|
|
|
|
|
|
struct ngx_http_upstream_check_peer_s { /*peer关键配置类:包括一组回调函数*/
|
|
|
|
|
struct ngx_http_upstream_check_peer_s {
|
|
|
|
|
ngx_flag_t state;
|
|
|
|
|
ngx_pool_t *pool;
|
|
|
|
|
ngx_uint_t index;
|
|
|
|
@ -125,17 +125,17 @@ struct ngx_http_upstream_check_peer_s { /*peer关键配置类:包括一组回
|
|
|
|
|
ngx_str_t *upstream_name;
|
|
|
|
|
ngx_addr_t *check_peer_addr;
|
|
|
|
|
ngx_addr_t *peer_addr;
|
|
|
|
|
ngx_event_t check_ev; /*关键数据结构:check event*/
|
|
|
|
|
ngx_event_t check_timeout_ev; /*关键数据结构: check timeout event*/
|
|
|
|
|
ngx_peer_connection_t pc;
|
|
|
|
|
ngx_event_t check_ev; /* 主动探测事件 */
|
|
|
|
|
ngx_event_t check_timeout_ev; /* 主动探测超时事件 */
|
|
|
|
|
ngx_peer_connection_t pc; /* 保存与后端服务器的连接 */
|
|
|
|
|
|
|
|
|
|
void *check_data;
|
|
|
|
|
ngx_event_handler_pt send_handler; /*TODO: ???*/
|
|
|
|
|
ngx_event_handler_pt recv_handler; /*TODO: ???*/
|
|
|
|
|
ngx_event_handler_pt send_handler; /* ngx_connection_t中的write事件handler:向后端服务器发送探测数据 */
|
|
|
|
|
ngx_event_handler_pt recv_handler; /* ngx_connection_t中的read事件handler:从后端服务器接收探测结果数据 */
|
|
|
|
|
|
|
|
|
|
ngx_http_upstream_check_packet_init_pt init; /*TODO: ???*/
|
|
|
|
|
ngx_http_upstream_check_packet_parse_pt parse; /*TODO: ???*/
|
|
|
|
|
ngx_http_upstream_check_packet_clean_pt reinit; /*TODO: ???*/
|
|
|
|
|
ngx_http_upstream_check_packet_init_pt init;
|
|
|
|
|
ngx_http_upstream_check_packet_parse_pt parse;
|
|
|
|
|
ngx_http_upstream_check_packet_clean_pt reinit;
|
|
|
|
|
|
|
|
|
|
ngx_http_upstream_check_peer_shm_t *shm;
|
|
|
|
|
ngx_http_upstream_check_srv_conf_t *conf;
|
|
|
|
@ -163,7 +163,7 @@ typedef struct {
|
|
|
|
|
#define NGX_CHECK_HTTP_5XX 0x0010
|
|
|
|
|
#define NGX_CHECK_HTTP_ERR 0x8000
|
|
|
|
|
|
|
|
|
|
typedef struct {/*核心配置:包括了check相关的回调函数*/
|
|
|
|
|
typedef struct {
|
|
|
|
|
ngx_uint_t type;
|
|
|
|
|
|
|
|
|
|
ngx_str_t name;
|
|
|
|
@ -173,12 +173,12 @@ typedef struct {/*核心配置:包括了check相关的回调函数*/
|
|
|
|
|
/* HTTP */
|
|
|
|
|
ngx_uint_t default_status_alive;
|
|
|
|
|
|
|
|
|
|
ngx_event_handler_pt send_handler; /*TODO: ???*/
|
|
|
|
|
ngx_event_handler_pt recv_handler; /*TODO: ???*/
|
|
|
|
|
ngx_event_handler_pt send_handler;
|
|
|
|
|
ngx_event_handler_pt recv_handler;
|
|
|
|
|
|
|
|
|
|
ngx_http_upstream_check_packet_init_pt init; /*TODO: ???*/
|
|
|
|
|
ngx_http_upstream_check_packet_parse_pt parse; /*TODO: ???*/
|
|
|
|
|
ngx_http_upstream_check_packet_clean_pt reinit; /*TODO: ???*/
|
|
|
|
|
ngx_http_upstream_check_packet_init_pt init;
|
|
|
|
|
ngx_http_upstream_check_packet_parse_pt parse;
|
|
|
|
|
ngx_http_upstream_check_packet_clean_pt reinit;
|
|
|
|
|
|
|
|
|
|
unsigned need_pool;
|
|
|
|
|
unsigned need_keepalive;
|
|
|
|
@ -941,9 +941,9 @@ ngx_http_upstream_check_free_peer(ngx_uint_t index)
|
|
|
|
|
ngx_shmtx_unlock(&peer[index].shm->mutex);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/*核心函数:根据当前配置,加载peer对象,为各个peer对象注册回调函数*/
|
|
|
|
|
static ngx_int_t
|
|
|
|
|
ngx_http_upstream_check_add_timers(ngx_cycle_t *cycle)
|
|
|
|
|
ngx_http_upstream_check_add_timers(ngx_cycle_t *cycle) /*此函数被调用的时间是:master进程创建出worker进程时,在初始化当前module的cycle对象时,被调用*/
|
|
|
|
|
{
|
|
|
|
|
ngx_uint_t i;
|
|
|
|
|
ngx_msec_t t, delay;
|
|
|
|
@ -978,7 +978,7 @@ ngx_http_upstream_check_add_timers(ngx_cycle_t *cycle)
|
|
|
|
|
for (i = 0; i < peers->peers.nelts; i++) {
|
|
|
|
|
peer[i].shm = &peer_shm[i];
|
|
|
|
|
|
|
|
|
|
peer[i].check_ev.handler = ngx_http_upstream_check_begin_handler; /* TODO 整个回调函数处理什么事件event?*/
|
|
|
|
|
peer[i].check_ev.handler = ngx_http_upstream_check_begin_handler; /* 回调函数:处理定时check后端服务器的事件*/
|
|
|
|
|
peer[i].check_ev.log = cycle->log;
|
|
|
|
|
peer[i].check_ev.data = &peer[i];
|
|
|
|
|
peer[i].check_ev.timer_set = 0;
|
|
|
|
@ -999,8 +999,8 @@ ngx_http_upstream_check_add_timers(ngx_cycle_t *cycle)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
peer[i].send_handler = cf->send_handler;
|
|
|
|
|
peer[i].recv_handler = cf->recv_handler;
|
|
|
|
|
peer[i].send_handler = cf->send_handler; /* send event handler*/
|
|
|
|
|
peer[i].recv_handler = cf->recv_handler; /* recv event handler*/
|
|
|
|
|
|
|
|
|
|
peer[i].init = cf->init;
|
|
|
|
|
peer[i].parse = cf->parse;
|
|
|
|
@ -1013,13 +1013,13 @@ ngx_http_upstream_check_add_timers(ngx_cycle_t *cycle)
|
|
|
|
|
delay = ucscf->check_interval > 1000 ? ucscf->check_interval : 1000;
|
|
|
|
|
t = ngx_random() % delay;
|
|
|
|
|
|
|
|
|
|
ngx_add_timer(&peer[i].check_ev, t);
|
|
|
|
|
ngx_add_timer(&peer[i].check_ev, t); /*为每个后端服务器(即peer)添加timer:nginx后端会计时,时间到就触发事件的回调函数*/
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return NGX_OK;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* 处理check相关event*/
|
|
|
|
|
/* timer event:定期check的事件*/
|
|
|
|
|
static void
|
|
|
|
|
ngx_http_upstream_check_begin_handler(ngx_event_t *event)
|
|
|
|
|
{
|
|
|
|
@ -1055,7 +1055,7 @@ ngx_http_upstream_check_begin_handler(ngx_event_t *event)
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interval = ngx_current_msec - peer->shm->access_time;
|
|
|
|
|
interval = ngx_current_msec - peer->shm->access_time; /* 距离上一次检测过去了多少毫秒 */
|
|
|
|
|
ngx_log_debug5(NGX_LOG_DEBUG_HTTP, event->log, 0,
|
|
|
|
|
"http check begin handler index: %ui, owner: %P, "
|
|
|
|
|
"ngx_pid: %P, interval: %M, check_interval: %M",
|
|
|
|
@ -1111,7 +1111,7 @@ ngx_http_upstream_check_connect_handler(ngx_event_t *event)
|
|
|
|
|
peer = event->data;
|
|
|
|
|
ucscf = peer->conf;
|
|
|
|
|
|
|
|
|
|
if (peer->pc.connection != NULL) {
|
|
|
|
|
if (peer->pc.connection != NULL) { /* 已与后端服务器建立连接 */
|
|
|
|
|
c = peer->pc.connection;
|
|
|
|
|
if ((rc = ngx_http_upstream_check_peek_one_byte(c)) == NGX_OK) {
|
|
|
|
|
goto upstream_check_connect_done;
|
|
|
|
@ -1122,20 +1122,20 @@ ngx_http_upstream_check_connect_handler(ngx_event_t *event)
|
|
|
|
|
}
|
|
|
|
|
ngx_memzero(&peer->pc, sizeof(ngx_peer_connection_t));
|
|
|
|
|
|
|
|
|
|
peer->pc.sockaddr = peer->check_peer_addr->sockaddr; /*TODO 挑选后端服务器*/
|
|
|
|
|
peer->pc.sockaddr = peer->check_peer_addr->sockaddr;
|
|
|
|
|
peer->pc.socklen = peer->check_peer_addr->socklen;
|
|
|
|
|
peer->pc.name = &peer->check_peer_addr->name;
|
|
|
|
|
|
|
|
|
|
peer->pc.get = ngx_event_get_peer;
|
|
|
|
|
peer->pc.get = ngx_event_get_peer; /* 如何挑选后端服务器?由Nginx内置的轮询(round robin)和IP哈希(ip hash)模块提供! */
|
|
|
|
|
peer->pc.log = event->log;
|
|
|
|
|
peer->pc.log_error = NGX_ERROR_ERR;
|
|
|
|
|
|
|
|
|
|
peer->pc.cached = 0;
|
|
|
|
|
peer->pc.connection = NULL;
|
|
|
|
|
|
|
|
|
|
rc = ngx_event_connect_peer(&peer->pc);
|
|
|
|
|
rc = ngx_event_connect_peer(&peer->pc); /* 连接后端服务器,等待触发write和send事件的回调函数 */
|
|
|
|
|
|
|
|
|
|
if (rc == NGX_ERROR || rc == NGX_DECLINED) {
|
|
|
|
|
if (rc == NGX_ERROR || rc == NGX_DECLINED) { /* 检测后端服务器失败 */
|
|
|
|
|
ngx_http_upstream_check_status_update(peer, 0);
|
|
|
|
|
ngx_http_upstream_check_clean_event(peer);
|
|
|
|
|
return;
|
|
|
|
@ -1171,7 +1171,7 @@ ngx_http_upstream_check_peek_one_byte(ngx_connection_t *c)
|
|
|
|
|
ngx_int_t n;
|
|
|
|
|
ngx_err_t err;
|
|
|
|
|
|
|
|
|
|
n = recv(c->fd, buf, 1, MSG_PEEK);
|
|
|
|
|
n = recv(c->fd, buf, 1, MSG_PEEK); /* 系统调用:从指定socket接收数据 */
|
|
|
|
|
err = ngx_socket_errno;
|
|
|
|
|
|
|
|
|
|
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, err,
|
|
|
|
@ -1315,7 +1315,7 @@ ngx_http_upstream_check_send_handler(ngx_event_t *event)
|
|
|
|
|
goto check_send_fail;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (peer->init == NULL || peer->init(peer) != NGX_OK) {
|
|
|
|
|
if (peer->init == NULL || peer->init(peer) != NGX_OK) { /* 初始化peer:调用ngx_http_upstream_check_http_init */
|
|
|
|
|
|
|
|
|
|
ngx_log_error(NGX_LOG_ERR, event->log, 0,
|
|
|
|
|
"check init error with peer: %V ",
|
|
|
|
@ -1329,7 +1329,7 @@ ngx_http_upstream_check_send_handler(ngx_event_t *event)
|
|
|
|
|
|
|
|
|
|
while (ctx->send.pos < ctx->send.last) {
|
|
|
|
|
|
|
|
|
|
size = c->send(c, ctx->send.pos, ctx->send.last - ctx->send.pos);
|
|
|
|
|
size = c->send(c, ctx->send.pos, ctx->send.last - ctx->send.pos); /* 向后端服务器发送探测数据 */
|
|
|
|
|
|
|
|
|
|
#if (NGX_DEBUG)
|
|
|
|
|
{
|
|
|
|
@ -1360,14 +1360,14 @@ ngx_http_upstream_check_send_handler(ngx_event_t *event)
|
|
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
check_send_fail:
|
|
|
|
|
check_send_fail: /*心跳数据发送失败*/
|
|
|
|
|
ngx_http_upstream_check_status_update(peer, 0);
|
|
|
|
|
ngx_http_upstream_check_clean_event(peer);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
ngx_http_upstream_check_recv_handler(ngx_event_t *event)
|
|
|
|
|
ngx_http_upstream_check_recv_handler(ngx_event_t *event) /*处理后端服务器返回的响应数据*/
|
|
|
|
|
{
|
|
|
|
|
u_char *new_buf;
|
|
|
|
|
ssize_t size, n;
|
|
|
|
@ -1381,7 +1381,7 @@ ngx_http_upstream_check_recv_handler(ngx_event_t *event)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
c = event->data;
|
|
|
|
|
peer = c->data;
|
|
|
|
|
peer = c->data; /*响应数据被nginx封装在event中:rev event的data中,是connection类型,connection的data中是自定义的peer数据(如何关联上的呢?在ngx_http_upstream_check_begin_handler处理check事件时),peer中有check上下文数据,其中就包含了send和recv数据*/
|
|
|
|
|
|
|
|
|
|
if (peer->state != NGX_HTTP_CHECK_SEND_DONE) {
|
|
|
|
|
|
|
|
|
@ -1392,7 +1392,7 @@ ngx_http_upstream_check_recv_handler(ngx_event_t *event)
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ctx = peer->check_data;
|
|
|
|
|
ctx = peer->check_data; /*健康检测上下文*/
|
|
|
|
|
|
|
|
|
|
if (ctx->recv.start == NULL) {
|
|
|
|
|
/* 1/2 of the page_size, is it enough? */
|
|
|
|
@ -1449,7 +1449,7 @@ ngx_http_upstream_check_recv_handler(ngx_event_t *event)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rc = peer->parse(peer);
|
|
|
|
|
rc = peer->parse(peer); /*解析响应数据:调用ngx_http_upstream_check_http_parse*/
|
|
|
|
|
|
|
|
|
|
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
|
|
|
|
|
"http check parse rc: %i, peer: %V ",
|
|
|
|
@ -2516,19 +2516,19 @@ ngx_http_upstream_check_status_update(ngx_http_upstream_check_peer_t *peer,
|
|
|
|
|
|
|
|
|
|
ucscf = peer->conf;
|
|
|
|
|
|
|
|
|
|
if (result) {
|
|
|
|
|
if (result) { /* 1 检测成功 */
|
|
|
|
|
peer->shm->rise_count++;
|
|
|
|
|
peer->shm->fall_count = 0;
|
|
|
|
|
if (peer->shm->down && peer->shm->rise_count >= ucscf->rise_count) {
|
|
|
|
|
if (peer->shm->down && peer->shm->rise_count >= ucscf->rise_count) { /* 成功次数达到复活阈值,复活后端服务器 */
|
|
|
|
|
peer->shm->down = 0;
|
|
|
|
|
ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0,
|
|
|
|
|
"enable check peer: %V ",
|
|
|
|
|
&peer->check_peer_addr->name);
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
} else { /* 0 检测失败 */
|
|
|
|
|
peer->shm->rise_count = 0;
|
|
|
|
|
peer->shm->fall_count++;
|
|
|
|
|
if (!peer->shm->down && peer->shm->fall_count >= ucscf->fall_count) {
|
|
|
|
|
if (!peer->shm->down && peer->shm->fall_count >= ucscf->fall_count) { /* 失败次数达到关停阈值,关停后端服务器 */
|
|
|
|
|
peer->shm->down = 1;
|
|
|
|
|
ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0,
|
|
|
|
|
"disable check peer: %V ",
|
|
|
|
@ -2536,7 +2536,7 @@ ngx_http_upstream_check_status_update(ngx_http_upstream_check_peer_t *peer,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
peer->shm->access_time = ngx_current_msec;
|
|
|
|
|
peer->shm->access_time = ngx_current_msec; /* 记录检测时间,以毫秒为单位*/
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -2574,7 +2574,7 @@ ngx_http_upstream_check_clean_event(ngx_http_upstream_check_peer_t *peer)
|
|
|
|
|
peer->state = NGX_HTTP_CHECK_ALL_DONE;
|
|
|
|
|
|
|
|
|
|
if (peer->check_data != NULL && peer->reinit) {
|
|
|
|
|
peer->reinit(peer);
|
|
|
|
|
peer->reinit(peer); /*重新初始化peer:调用ngx_http_upstream_check_http_reinit函数*/
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
peer->shm->owner = NGX_INVALID_PID;
|
|
|
|
@ -2614,7 +2614,7 @@ ngx_http_upstream_check_finish_handler(ngx_event_t *event)
|
|
|
|
|
static ngx_int_t
|
|
|
|
|
ngx_http_upstream_check_need_exit()
|
|
|
|
|
{
|
|
|
|
|
if (ngx_terminate || ngx_exiting || ngx_quit) {
|
|
|
|
|
if (ngx_terminate || ngx_exiting || ngx_quit) { /* 当nginx正在终止时,清空所有相关event */
|
|
|
|
|
ngx_http_upstream_check_clear_all_events();
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
@ -2624,7 +2624,7 @@ ngx_http_upstream_check_need_exit()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
ngx_http_upstream_check_clear_all_events()
|
|
|
|
|
ngx_http_upstream_check_clear_all_events() /* 删除timer,关闭connection,销毁peer的内存pool */
|
|
|
|
|
{
|
|
|
|
|
ngx_uint_t i;
|
|
|
|
|
ngx_connection_t *c;
|
|
|
|
@ -3089,7 +3089,7 @@ ngx_http_upstream_check(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
|
|
|
|
|
|
|
|
|
|
value = cf->args->elts;/*数组指针,指向:{{len = 5, data = 0x555555706795 "check"}, {len = 13, data = 0x55555570679b "interval=1000"}, {len = 6, data = 0x5555557067a9 "rise=1"}, {len = 6, data = 0x5555557067b0 "fall=3"}, {len = 12, data = 0x5555557067b7 "timeout=2000"}, {len = 9, data = 0x5555557067c4 "type=http"}}*/
|
|
|
|
|
|
|
|
|
|
/* 获取已经注册的自定义configuration结构体, TODO: 那么,在哪里注册的configuration结构体呢?*/
|
|
|
|
|
/* 获取已经注册的自定义configuration结构体, 那么,在哪里注册的configuration结构体呢?在ngx_http_upstream_check_module_ctx变量中*/
|
|
|
|
|
ucscf = ngx_http_conf_get_module_srv_conf(cf,
|
|
|
|
|
ngx_http_upstream_check_module);
|
|
|
|
|
if (ucscf == NULL) {
|
|
|
|
@ -3754,7 +3754,7 @@ ngx_http_upstream_check_init_shm(ngx_conf_t *cf, void *conf)
|
|
|
|
|
shm_name, shm_size);
|
|
|
|
|
|
|
|
|
|
shm_zone->data = cf->pool;
|
|
|
|
|
check_peers_ctx = ucmcf->peers;
|
|
|
|
|
check_peers_ctx = ucmcf->peers; /*获取后端服务器列表*/
|
|
|
|
|
|
|
|
|
|
shm_zone->init = ngx_http_upstream_check_init_shm_zone;
|
|
|
|
|
}
|
|
|
|
|