13.prometheus监控tengine（无用）

一、环境准备

1.1 docker安装tengine带nginx-module-vts模块(二选一)

mkdir /data/ -p
cd /data/

# 通过git clone下载已经创建好的docker-compose.yaml文件
 git clone https://gitee.com/linge365/docker-compose.git

version: '3'
services:
  tengine:
    image: registry.cn-shenzhen.aliyuncs.com/linge365/tengine:2.4.0-geoip2
    container_name: tengine               # 容器名,相当于docker run命令中的--name
    restart: unless-stopped               # 之前是什么状态，docker重启后，保持之前的状态（如果之前是stop，那docker重启时，也是stop状态）
    volumes:                              # 数据卷挂载路径设置,将本机目录映射到容器目录,相当于docker run命令中的-v
      - ./conf/nginx.conf:/etc/nginx/nginx.conf
      - ./conf/conf.d:/etc/nginx/conf.d
      - ./html:/usr/share/nginx/html
      - ./log:/var/log/nginx
      - ./cert:/etc/nginx/cert
      #- ./geoip2:/etc/nginx/geoip2
    environment:                        # 设置环境变量,相当于docker run命令中的-e
      TZ: Asia/Shanghai
      LANG: en_US.UTF-8
    ports:                              # 映射端口，相当于docker run 命令中的-p
      - "80:80"
      - "443:443"

tengine.yaml文件

cd docker-compose/tengine
docker-compose up -d

1.2 编译安装的tengine安装nginx-module-vts模块（二选一）

# 如果tengine是自行编译安装的，安装nginx-module-vts模块，如下
#下载nginx-module-vts插件

cd /usr/local/src/
git clone git://github.com/vozlt/nginx-module-vts.git

# 重新编译nginx，指定nginx-module-vts目录(编译前记得备份原来的tengine目录)
cd /usr/local/src/tengine-2.4.0  #进入到tengine源码目录
./configure  ...  --add-module=/usr/local/src/nginx-module-vts
make
make install

二、监控tengine

2.1 准备tengine环境

tengine开启status(docker安装的tengine)

注：
●监控tengine我将使用第三方nginx-module-vts模块。
检查是否安装有nginx-module-vts模块，nginx-module-vts模块已在上面的环境搭建安装好。

[root@test tengine]# docker exec -it tengine nginx -V 2>&1 | grep -o nginx-module-vts
nginx-module-vts

修改配置文件

cd /data/tengine/docker-compose/tengine
vim conf/nginx.conf
# 在http标签中加入vhost_traffic_status_zone配置
http {
    vhost_traffic_status_zone;

    ...
}

vim conf/nginx_prod.conf
http {
    vhost_traffic_status_zone;

    ...

    server {

        ...

        location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;
        }
    }
}

vim conf/conf.d/default.conf
    server {

        ...

        location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;
        }
    }

# 检查配置
[root@test tengine]# docker exec -it tengine nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful

# 重启
docker restart tengine

# web检查
curl http://192.168.11.62/status/format/json
http://192.168.10.100/status

2.2 二进制安装nginx-vts-exporter(二选一)

nginx-vts-exporter下载地址: https://github.com/hnlq715/nginx-vts-exporter/releases

wget https://github.com/hnlq715/nginx-vts-exporter/releases/download/v0.10.8/nginx-vtx-exporter_0.10.8_linux_amd64.tar.gz

mkdir /opt/prometheus/nginx_vtx_exporter -p
tar xvf nginx-vtx-exporter_0.10.8_linux_amd64.tar.gz -C /opt/prometheus/nginx_vtx_exporter
ls -l /opt/prometheus/nginx_vtx_exporter

# 创建用户，修改文件夹权限
useradd -M -s /usr/sbin/nologin prometheus
chown prometheus:prometheus -R /opt/prometheus

# 创建systemd服务
cat > /etc/systemd/system/nginx_vtx_exporter.service <<"EOF"
[Unit]
Description=nginx_vtx_exporter
After=network.target

[Service]
Type=simple
User=prometheus
Group=prometheus
Restart=always
ExecStart=/opt/prometheus/nginx_vtx_exporter/nginx-vtx-exporter -nginx.scrape_uri=http://192.168.11.62/status/format/json

[Install]
WantedBy=multi-user.target
EOF

# 启动服务
systemctl daemon-reload
systemctl start nginx_vtx_exporter.service
systemctl enable nginx_vtx_exporter.service

2.3 docker安装nginx-vts-exporter

docker-compose方式

mkdir /data/nginx_vts_exporter

cd /data/nginx_vts_exporter

cat >docker-compose.yaml <<EOF
version: '3.3'
services:
  nginx_vts_exporter:
    image: sophos/nginx-vts-exporter:latest
    container_name: nginx_vts_exporter
    environment:
      NGINX_STATUS: http://192.168.11.62/status/format/json
    restart: always
    ports:
      - "9913:9913"
EOF

# 启动
docker-compose up -d

# metrics地址
http://192.168.10.100:9913/metrics

3.Prometheus配置

cd /data/docker-prometheus 

#在scrape_configs(搜刮配置):下面增加如下配置：

cat >> prometheus/prometheus.yml << "EOF"
  - job_name: 'nginx_vts_exporter'
    static_configs:
    - targets: ['192.168.10.100:9913']
      labels:
        instance: test服务器
EOF

# 重新加载配置
curl -X POST http://localhost:9090/-/reload

# 检查
http://192.168.10.14:9090/targets

3.1 常用监控指标

nginx_server_connections{status="accepted"} 接收请求数
nginx_server_connections{status="active"} 活动连接数
nginx_server_connections{status="handled"} 成功处理请求数
nginx_server_connections{status="reading"} 正在进行读操作的请求数
nginx_server_connections{status="requests"} 总请求数
nginx_server_connections{status="waiting"} 正在等待的请求数
nginx_server_connections{status="writing"} 正在进行写操作的请求数

nginx_upstream_requests{backend="xx:8080",code="4xx"} 后端节点请求状态

nginx_server_requests{code="4xx",host="*"}          状态码为4xx,所有主机
nginx_server_requests{code="4xx",host="localhost"}  状态码为4xx,localhost主机
nginx_server_requests{code="5xx",host="*"}          状态码为5xx,所有主机
nginx_server_requests{code="5xx",host="localhost"}  状态码为5xx,localhost主机

4. 触发器

cat >>prometheus/rules/tengine.yml <<"EOF"
groups:
- name: tengine
  rules:
  - alert: 客户端请求tengine非200
    expr: sum by (code) (rate(nginx_server_requests{code=~"4xx|5xx"}[2m])) > 0
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: "客户端http请求异常"
      description: "HTTP状态码非 200-399"
  - alert: tengine请求后端节点非200
    expr: sum by (code) (rate(nginx_upstream_requests{code=~"4xx|5xx"}[2m])) > 0
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: "后端节点http请求异常"
      description: "后端节点请求状态码非 200-399"
EOF

注

• 2个触发器大部分重复（如：tengine请求后端节点返回500，那tengine也会给客户端返回500）

检查配置并加载

docker exec -it prometheus promtool check config /etc/prometheus/prometheus.yml

curl -X POST http://localhost:9090/-/reload

# 检查
http://192.168.10.14:9090/rules
http://192.168.10.14:9090/alerts?search=

5. dashboard

grafana展示prometheus从nginx_exporter收集到的的数据

https://grafana.com/grafana/dashboards/2949