http代理阅读2

向上游服务器发送请求处理

static void
ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u,
    ngx_uint_t do_write) //向上游服务器发送请求   当一次发送不完，通过ngx_http_upstream_send_request_handler再次触发发送
{
    ngx_int_t          rc;
    ngx_connection_t  *c;

    c = u->peer.connection; //向上游服务器的连接信息

    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
                   "http upstream send request");

    if (u->state->connect_time == (ngx_msec_t) -1) {//更新upstream 连接时间耗费状态信息
        u->state->connect_time = ngx_current_msec - u->state->response_time;
    }

    /*通过getsockopt测试与上游服务器的tcp连接是否异常
         * BSDs and Linux return 0 and set a pending error in err
         * Solaris returns -1 and sets errno
        if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, (void *) &err, &len) == -1)*/
    if (!u->request_sent && ngx_http_upstream_test_connect(c) != NGX_OK) { //测试连接失败
        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);//如果测试失败，调用ngx_http_upstream_next函数，这个函数可能再次调用peer.get调用别的连接。
        return;
    }

    c->log->action = "sending request to upstream";

    rc = ngx_http_upstream_send_request_body(r, u, do_write);

    if (rc == NGX_ERROR) {
        /*  若返回值rc=NGX_ERROR，表示当前连接上出错， 将错误信息传递给ngx_http_upstream_next方法， 该方法根据错误信息决定
        是否重新向上游其他服务器发起连接； 并return从当前函数返回； */
        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);
        return;
    }

    if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {//满足http错误状态码条件 结束请求 同时响应结果到请求端
        ngx_http_upstream_finalize_request(r, u, rc);
        return;
    }

    /*
         若返回值rc = NGX_AGAIN，表示请求数据并未完全发送， 即有剩余的请求数据保存在output中，但此时，写事件已经不可写，
         则调用ngx_add_timer方法把当前连接上的写事件添加到定时器机制， 并调用ngx_handle_write_event方法将写事件注册到epoll事件机制中；
     */ //通过ngx_http_upstream_read_request_handler进行再次epoll write
    if (rc == NGX_AGAIN) {//协议栈缓冲区已满，需要等待发送数据出去后出发epoll可写，从而继续write
        if (!c->write->ready) {
        //这里加定时器的原因是，例如我把数据扔到协议栈了，并且协议栈已经满了，但是对方就是不接受数据，造成数据一直在协议栈缓存中
        //因此只要数据发送出去，就会触发epoll继续写，从而在下面两行删除写超时定时器
            ngx_add_timer(c->write, u->conf->send_timeout, NGX_FUNC_LINE);
            //如果超时会执行ngx_http_upstream_send_request_handler，这里面对写超时进行处理

        } else if (c->write->timer_set) { //例如ngx_http_upstream_send_request_body发送了三次返回NGX_AGAIN,那么第二次就需要把第一次上面的超时定时器关了，表示发送正常
            ngx_del_timer(c->write, NGX_FUNC_LINE);
        }

        //在连接后端服务器conncet前，有设置ngx_add_conn，里面已经将fd添加到了读写事件中，因此这里实际上只是简单执行下ngx_send_lowat
        if (ngx_handle_write_event(c->write, u->conf->send_lowat, NGX_FUNC_LINE) != NGX_OK) {
            ngx_http_upstream_finalize_request(r, u,
                                               NGX_HTTP_INTERNAL_SERVER_ERROR);
            return;
        }

        return;
    }

    /* rc == NGX_OK */
    //向后端的数据发送完毕

    //当发往后端服务器的数据包过大，需要分多次发送的时候，在上面的if (rc == NGX_AGAIN)中会添加定时器来触发发送，如果协议栈一直不发送数据出去
    //就会超时，如果数据最终全部发送出去则需要为最后一次time_write添加删除操作。

    //如果发往后端的数据长度后小，则一般不会再上门添加定时器，这里的timer_set肯定为0，所以如果拔掉后端网线，通过ngx_http_upstream_test_connect
    //是判断不出后端服务器掉线的，上面的ngx_http_upstream_send_request_body还是会返回成功的，所以这里有个bug
    if (c->write->timer_set) { //这里的定时器是ngx_http_upstream_connect中connect返回NGX_AGAIN的时候添加的定时器
        /*
2025/04/24 02:54:29[             ngx_event_connect_peer,    32]  [debug] 14867#14867: *1 socket 12
2025/04/24 02:54:29[           ngx_epoll_add_connection,  1486]  [debug] 14867#14867: *1 epoll add connection: fd:12 ev:80002005
2025/04/24 02:54:29[             ngx_event_connect_peer,   125]  [debug] 14867#14867: *1 connect to 127.0.0.1:3666, fd:12 #2
2025/04/24 02:54:29[          ngx_http_upstream_connect,  1549]  [debug] 14867#14867: *1 http upstream connect: -2 //返回NGX_AGAIN
2025/04/24 02:54:29[                ngx_event_add_timer,    88]  [debug] 14867#14867: *1 <ngx_http_upstream_connect,  1665>  event timer add: 12: 60000:1677807811 //这里添加
2025/04/24 02:54:29[          ngx_http_finalize_request,  2526]  [debug] 14867#14867: *1 http finalize request: -4, "/test.php?" a:1, c:2
2025/04/24 02:54:29[             ngx_http_close_request,  3789]  [debug] 14867#14867: *1 http request count:2 blk:0
2025/04/24 02:54:29[           ngx_worker_process_cycle,  1110]  [debug] 14867#14867: worker(14867) cycle again
2025/04/24 02:54:29[           ngx_trylock_accept_mutex,   405]  [debug] 14867#14867: accept mutex locked
2025/04/24 02:54:29[           ngx_epoll_process_events,  1614]  [debug] 14867#14867: begin to epoll_wait, epoll timer: 60000
2025/04/24 02:54:29[           ngx_epoll_process_events,  1699]  [debug] 14867#14867: epoll: fd:11 epoll-out(ev:0004) d:B27440E8
2025/04/24 02:54:29[           ngx_epoll_process_events,  1772]  [debug] 14867#14867: *1 post event AEB44068
2025/04/24 02:54:29[           ngx_epoll_process_events,  1699]  [debug] 14867#14867: epoll: fd:12 epoll-out(ev:0004) d:B2744158
2025/04/24 02:54:29[           ngx_epoll_process_events,  1772]  [debug] 14867#14867: *1 post event AEB44098
2025/04/24 02:54:29[      ngx_process_events_and_timers,   371]  [debug] 14867#14867: epoll_wait timer range(delta): 2
2025/04/24 02:54:29[           ngx_event_process_posted,    65]  [debug] 14867#14867: posted event AEB44068
2025/04/24 02:54:29[           ngx_event_process_posted,    67]  [debug] 14867#14867: *1 delete posted event AEB44068
2025/04/24 02:54:29[           ngx_http_request_handler,  2400]  [debug] 14867#14867: *1 http run request: "/test.php?"
2025/04/24 02:54:29[ngx_http_upstream_check_broken_connection,  1335]  [debug] 14867#14867: *1 http upstream check client, write event:1, "/test.php"
2025/04/24 02:54:29[ngx_http_upstream_check_broken_connection,  1458]  [debug] 14867#14867: *1 http upstream recv(): -1 (11: Resource temporarily unavailable)
2025/04/24 02:54:29[           ngx_event_process_posted,    65]  [debug] 14867#14867: posted event AEB44098
2025/04/24 02:54:29[           ngx_event_process_posted,    67]  [debug] 14867#14867: *1 delete posted event AEB44098
2025/04/24 02:54:29[          ngx_http_upstream_handler,  1295]  [debug] 14867#14867: *1 http upstream request: "/test.php?"
2025/04/24 02:54:29[ngx_http_upstream_send_request_handler,  2210]  [debug] 14867#14867: *1 http upstream send request handler
2025/04/24 02:54:29[     ngx_http_upstream_send_request,  2007]  [debug] 14867#14867: *1 http upstream send request
2025/04/24 02:54:29[ngx_http_upstream_send_request_body,  2095]  [debug] 14867#14867: *1 http upstream send request body
2025/04/24 02:54:29[                   ngx_chain_writer,   690]  [debug] 14867#14867: *1 chain writer buf fl:0 s:968
2025/04/24 02:54:29[                   ngx_chain_writer,   704]  [debug] 14867#14867: *1 chain writer in: 080EC838
2025/04/24 02:54:29[                         ngx_writev,   192]  [debug] 14867#14867: *1 writev: 968 of 968
2025/04/24 02:54:29[                   ngx_chain_writer,   740]  [debug] 14867#14867: *1 chain writer out: 00000000
2025/04/24 02:54:29[                ngx_event_del_timer,    39]  [debug] 14867#14867: *1 <ngx_http_upstream_send_request,  2052>  event timer del: 12: 1677807811//这里删除
2025/04/24 02:54:29[                ngx_event_add_timer,    88]  [debug] 14867#14867: *1 <ngx_http_upstream_send_request,  2075>  event timer add: 12: 60000:1677807813
           */
        ngx_del_timer(c->write, NGX_FUNC_LINE);
    }

    /* 若返回值 rc = NGX_OK，表示已经发送完全部请求数据， 准备接收来自上游服务器的响应报文，则执行以下程序；  */
    if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) {
        if (ngx_tcp_push(c->fd) == NGX_ERROR) {
            ngx_log_error(NGX_LOG_CRIT, c->log, ngx_socket_errno,
                          ngx_tcp_push_n " failed");
            ngx_http_upstream_finalize_request(r, u,
                                               NGX_HTTP_INTERNAL_SERVER_ERROR);
            return;
        }

        c->tcp_nopush = NGX_TCP_NOPUSH_UNSET;
    }

    u->write_event_handler = ngx_http_upstream_dummy_handler; //数据已经在前面全部发往后端服务器了，所以不需要再做写处理

    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "send out chain data to uppeer server OK");
    //在连接后端服务器conncet前，有设置ngx_add_conn，里面已经将fd添加到了读写事件中，因此这里实际上只是简单执行下ngx_send_lowat
    if (ngx_handle_write_event(c->write, 0, NGX_FUNC_LINE) != NGX_OK) {
        ngx_http_upstream_finalize_request(r, u,
                                           NGX_HTTP_INTERNAL_SERVER_ERROR);
        return;
    }

    //这回数据已经发送了，可以准备接收了，设置接收后端应答的超时定时器。
    /*
        该定时器在收到后端应答数据后删除，见ngx_event_pipe
        if (rev->timer_set) {
            ngx_del_timer(rev, NGX_FUNC_LINE);
        }
     */
    ngx_add_timer(c->read, u->conf->read_timeout, NGX_FUNC_LINE); //如果超时在该函数检测ngx_http_upstream_process_header

    if (c->read->ready) { //读事件触发 准备处理http头部信息---此时c 表示链接上游服务器的connect可读，也就是上游服务器回复了响应报文
        ngx_http_upstream_process_header(r, u);
        return;
    }
}