kubernetes之部署dashboard 和heapster

部署dashboard之前，先确保traefik https方式部署成功，这样就可以通过 https 域名的方式访问dashboard，无需kube-proxy转发了。假设traefik-ingress https部署完成。

下载dashboard yaml文件

wget https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/alternative/kubernetes-dashboard.yaml

由于k8s开启了rbac认证，因此需要添加serviceaccount

[root@node-01 ~]# cat kubernetes-dashboard.yaml
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# ------------------- Dashboard Secrets ------------------- #

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-certs
  namespace: kube-system
type: Opaque

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-csrf
  namespace: kube-system
type: Opaque
data:
  csrf: ""

---
# ------------------- Dashboard Service Account ------------------- #

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kube-system

---
# ------------------- Dashboard Role & Role Binding ------------------- #

#kind: Role
#apiVersion: rbac.authorization.k8s.io/v1
#metadata:
#  name: kubernetes-dashboard-minimal
#  namespace: kube-system
#rules:
#  # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
#- apiGroups: [""]
#  resources: ["secrets"]
#  verbs: ["create"]
#  # Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
#- apiGroups: [""]
#  resources: ["configmaps"]
#  verbs: ["create"]
#  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
#- apiGroups: [""]
#  resources: ["secrets"]
#  resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
#  verbs: ["get", "update", "delete"]
#  # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
#- apiGroups: [""]
#  resources: ["configmaps"]
#  resourceNames: ["kubernetes-dashboard-settings"]
#  verbs: ["get", "update"]
#  # Allow Dashboard to get metrics from heapster.
#- apiGroups: [""]
#  resources: ["services"]
#  resourceNames: ["heapster"]
#  verbs: ["proxy"]
#- apiGroups: [""]
#  resources: ["services/proxy"]
#  resourceNames: ["heapster", "http:heapster:", "https:heapster:"]
#  verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1
#kind: RoleBinding
kind: ClusterRoleBinding
metadata:
  name: kubernetes-dashboard-minimal
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
#  kind: Role
  kind: ClusterRole
#  name: kubernetes-dashboard-minimal
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: kubernetes-dashboard
  namespace: kube-system

---
# ------------------- Dashboard Deployment ------------------- #

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kube-system
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
    spec:
      containers:
      - name: kubernetes-dashboard
        image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1
        ports:
        - containerPort: 8443
          protocol: TCP
        args:
          - --auto-generate-certificates
          # Uncomment the following line to manually specify Kubernetes API server Host
          # If not specified, Dashboard will attempt to auto discover the API server and connect
          # to it. Uncomment only if the default does not work.
          # - --apiserver-host=http://my-address:port
        volumeMounts:
        - name: kubernetes-dashboard-certs
          mountPath: /certs
          # Create on-disk volume to store exec logs
        - mountPath: /tmp
          name: tmp-volume
        livenessProbe:
          httpGet:
            scheme: HTTPS
            path: /
            port: 8443
          initialDelaySeconds: 30
          timeoutSeconds: 30
      volumes:
      - name: kubernetes-dashboard-certs
        secret:
          secretName: kubernetes-dashboard-certs
      - name: tmp-volume
        emptyDir: {}
      serviceAccountName: kubernetes-dashboard
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule

---
# ------------------- Dashboard Service ------------------- #

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kube-system
spec:
  ports:
    - port: 443
      targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard

配置ingress

[root@node-01 ~]# cat kubernetes-dashboard-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: kube-ui
  namespace: kube-system
spec:
  rules:
  - host: k8sui.ptengine.jp
    http:
      paths:
      - path: '/'
        backend:
          serviceName: kubernetes-dashboard
          servicePort: 443

添加本地host，测试。

1、使用kubernetes-dashboard-token的tocken登陆，先获取tocken，令牌方式登陆 

[root@node-01 ~]# kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kubernetes-dashboard-token|awk '{print $1}')|grep token:|awk '{print $2}'

2、创建一个名为admin的ServiceAccount并绑定名为cluster-admin的ClusterRole角色(该角色拥有集群最高权限)，使用下面的yaml文件创建admin用户并赋予他管理员权限，然后可以通过token登陆dashbaord。这种认证方式本质上是通过ServiceAccount的身份认证加上Bearer token请求API server的方式实现。

[root@node-01 ~]# cat admin-token.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: admin
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: admin
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin
  namespace: kube-system
  labels:
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile

[root@node-01 ~]# kubectl create -f admin-token.yaml
[root@node-01 ~]# kubectl get secret -n kube-system | grep admin
admin-token-422fl                                kubernetes.io/service-account-token   3      17s

通过如下的命令来获取admin ServiceAccount的token：

[root@node-01 ~]# kubectl describe secret/admin-token-422fl -n kube-system
Name:         admin-token-422fl
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: admin
              kubernetes.io/service-account.uid: ec5caa59-7142-11e9-aa9a-fad20acb9b00

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1025 bytes
namespace:  11 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi10b2tlbi00MjJmbCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6ImVjNWNhYTU5LTcxNDItMTFlOS1hYTlhLWZhZDIwYWNiOWIwMCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbiJ9.gXi0mToE0sct0soTeR_TLcDC5Xnr2xCZpvEn-VhE_hZX_QtzhqmgCcUy2wQmpjPoF6eku59dpQVp9WyBYY_rJaAY6HzB3Nzr3pZmDvNdj5Qe1QwxJadp38cqGs7Ao6EZg82wKoXqGI3481rU59BgbcbMeOO75d_e8iN7s64ErpJ25AAWIhfnNvHIJJUP0HoNU8uWbtrcCpceqm-gBY2-hKyqFH5dekMEdoz6GOH9w2xTYeF8Cl6d5xpQ8WcBJ60b7bSVV0PPlhVsswxkA0v95gDGj18rjrLoLJTc0rBOL4FwXOpMeyIO5y7HGXnHWWIL9gMInwoxGloxQJf7RWCRZw

如上，我们得到了该用户的token，dashboard登陆即可。

---

部署heapster

kubernetes 获取性能参数，默认使用 metric server 获取，通过修改kube-controller-manager.yaml，可以修改获取方式。此处介绍heapster。

1、 修改 /etc/kubernetes/manifests/kube-controller-manager.yaml 添加

--horizontal-pod-autoscaler-use-rest-clients=false

master节点需要全部修改。修改完后，重启kube-controller-manager，如果是kubeadm部署，容器会自动重启。

2、 部署的应用需要添加resource限制

resources:
  limits:
    cpu: 200m
    memory: 30Mi
  requests:
    cpu: 100m
    memory: 20Mi

3、 整合heapster 和 influxdb

在没有配置heapster和influxdb的情况下，pod的metric信息是无法获取到的，而早前版本K8S的HPA特性依赖的metric数据来源恰巧就是heapster和influxdb。heapster会在后面的版本中废弃。

准备yaml文件

# cat heapster-sa.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: heapster
  namespace: kube-system

# cat heapster-rbac.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:heapster
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system

# cat heapster-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: heapster
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: heapster
    spec:
      serviceAccountName: heapster
      containers:
      - name: heapster
        image: k8s.gcr.io/heapster-amd64:v1.4.2
        imagePullPolicy: IfNotPresent
        command:
        - /heapster
        - --source=kubernetes:https://kubernetes.default
        - --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086

上面配置source和influxdb有问题，下面会修改。

# cat heapster-service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: Heapster
  name: heapster
  namespace: kube-system
spec:
  ports:
  - port: 80
    targetPort: 8082
  selector:
         k8s-app: heapster

# cat influxdb-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: monitoring-influxdb
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: influxdb
    spec:
      containers:
      - name: influxdb
        image: k8s.gcr.io/heapster-influxdb-amd64:v1.3.3
        volumeMounts:
        - mountPath: /data
          name: influxdb-storage
      volumes:
      - name: influxdb-storage
        emptyDir: {}

# cat influxdb-service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: monitoring-influxdb
  name: monitoring-influxdb
  namespace: kube-system
spec:
  ports:
  - port: 8086
    targetPort: 8086
  selector:
    k8s-app: influxdb

检查heapster日志

[root@node-01 hpa]# kubectl logs -f heapster-76b4794779-d2vph -n kube-system
I0508 06:16:51.944854       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-bjksx
I0508 06:16:51.944890       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz
E0508 06:17:05.003857       1 kubelet.go:231] error while getting containers from Kubelet: failed to get all container stats from Kubelet URL "http://172.19.8.114:10255/stats/container/": Post http://172.19.8.114:10255/stats/container/: dial tcp 172.19.8.114:10255: getsockopt: connection refused

通过kubectl top 命令也获取不到结果

[root@node-01 ~]# kubectl top pod
W0508 15:25:57.588871    8939 top_pod.go:259] Metrics not available for pod default/my-nginx-6785b88976-7rrll, age: 3h32m13.588851424s
error: Metrics not available for pod default/my-nginx-6785b88976-7rrll, age: 3h32m13.588851424s
[root@node-01 ~]# kubectl top node
error: metrics not available yet

解决办法：

#在heapster-deployment.yaml 清单文件中进行如下修改
- --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true
- --sink=influxdb:http://monitoring-influxdb.kube-system.svc.cluster.local:8086

然后删除heapster重建

kubectl delete -f heapster-deployment.yaml
kubectl apply -f heapster-deployment.yaml

继续 。。。。。发现新问题

遇到403错误

[root@node-01 hpa]# kubectl logs -f heapster-699c6b684d-8sj2q -n kube-system
I0508 06:20:33.630699       1 heapster.go:72] /heapster --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true --sink=influxdb:http://monitoring-influxdb.kube-system.svc.cluster.local:8086
I0508 06:20:33.630780       1 heapster.go:73] Heapster version v1.4.2
I0508 06:20:33.631200       1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default" and version v1
I0508 06:20:33.631235       1 configs.go:62] Using kubelet port 10250
I0508 06:20:33.657061       1 influxdb.go:278] created influxdb sink with options: host:monitoring-influxdb.kube-system.svc.cluster.local:8086 user:root db:k8s
I0508 06:20:33.657100       1 heapster.go:196] Starting with InfluxDB Sink
I0508 06:20:33.657111       1 heapster.go:196] Starting with Metric Sink
I0508 06:20:33.666165       1 heapster.go:106] Starting heapster on port 8082
I0508 06:20:38.888431       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-bjksx
I0508 06:20:38.888461       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz
I0508 06:20:54.158646       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz
I0508 06:20:54.158676       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-bjksx
E0508 06:21:05.018631       1 kubelet.go:231] error while getting containers from Kubelet: failed to get all container stats from Kubelet URL "https://172.19.8.113:10250/stats/container/": request failed - "403 Forbidden", response: "Forbidden (user=system:serviceaccount:kube-system:heapster, verb=create, resource=nodes, subresource=stats)"

解决办法：

查看ClusterRole: system:heapster的权限，发现的确没有针对Resource: nodes/stats 的create权限

[root@node-01 hpa]# kubectl describe clusterrole system:heapster
Name:         system:heapster
Labels:       kubernetes.io/bootstrapping=rbac-defaults
Annotations:  rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:
  Resources               Non-Resource URLs  Resource Names  Verbs
  ---------               -----------------  --------------  -----
  events                  []                 []              [get list watch]
  namespaces              []                 []              [get list watch]
  nodes                   []                 []              [get list watch]
  pods                    []                 []              [get list watch]
  deployments.extensions  []                 []              [get list watch]

修改ClusterRole: system:heapster的权限

生成清单文件

kubectl get clusterrole system:heapster -o yaml > heapster_modify.yaml

修改文件，增加verbs:create权限，增加resources:nodes/stats

[root@node-01 hpa]# cat heapster_modify.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  creationTimestamp: "2019-05-06T06:24:10Z"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:heapster
  resourceVersion: "50"
  selfLink: /apis/rbac.authorization.k8s.io/v1/clusterroles/system%3Aheapster
  uid: 8f773f30-6fc7-11e9-991a-fa982e6ff600
rules:
- apiGroups:
  - ""
  resources:
  - events
  - namespaces
  - nodes
  - pods
  - nodes/stats  # 增加
  verbs:
  - create   #增加
  - get
  - list
  - watch
- apiGroups:
  - extensions
  resources:
  - deployments
  verbs:
  - get
  - list
  - watch

执行

kubectl apply -f heapster_modify.yaml

删除heapster重新部署

kubectl delete -f heapster-deployment.yaml
kubectl apply -f heapster-deployment.yaml

再次检查heapster日志

[root@node-01 hpa]# kubectl logs -f heapster-699c6b684d-n2ggr -n kube-system
I0508 06:39:28.987133       1 heapster.go:72] /heapster --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true --sink=influxdb:http://monitoring-influxdb.kube-system.svc.cluster.local:8086
I0508 06:39:28.987229       1 heapster.go:73] Heapster version v1.4.2
I0508 06:39:28.987560       1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default" and version v1
I0508 06:39:28.987589       1 configs.go:62] Using kubelet port 10250
I0508 06:39:29.012055       1 influxdb.go:278] created influxdb sink with options: host:monitoring-influxdb.kube-system.svc.cluster.local:8086 user:root db:k8s
I0508 06:39:29.012098       1 heapster.go:196] Starting with InfluxDB Sink
I0508 06:39:29.012120       1 heapster.go:196] Starting with Metric Sink
I0508 06:39:29.021905       1 heapster.go:106] Starting heapster on port 8082
I0508 06:40:05.166962       1 influxdb.go:241] Created database "k8s" on influxDB server at "monitoring-influxdb.kube-system.svc.cluster.local:8086”
I0508 06:39:54.519349       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz
I0508 06:40:04.062180       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-bjksx
I0508 06:40:04.062246       1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz

heapster默认30秒检查一次，因此需要等上30s才会收集到数据

[root@node-01 ~]# kubectl top nodes
NAME      CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
node-01   305m         7%     2421Mi          31%
node-02   242m         6%     1906Mi          24%
node-03   224m         5%     1760Mi          22%
node-04   77m          1%     693Mi           8%
node-05   82m          2%     848Mi           10%
node-06   87m          2%     677Mi           8%
[root@node-01 ~]# kubectl top pods
NAME                                CPU(cores)   MEMORY(bytes)
my-nginx-6785b88976-7rrll           0m           1Mi
nginx-deployment-6d6fdc59f7-bjksx   0m           1Mi
nginx-deployment-6d6fdc59f7-q4vjz   0m           1Mi

此时登录dashboard，可以单独内存、CPU信息。