## k8s接入Prometheus

**这里的 k8s 是使用 kubeadm 版**



```shell
[root@k8s-80 ~]# cd /k8syaml/

[root@k8s-80 k8syaml]# vim prometheus-ns.yaml
```

```yaml
apiVersion: v1
kind: Namespace
metadata:
  name: monitor
```



```
[root@k8s-80 k8syaml]# vim prometheus-cm.yaml
```

```yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitor
data:
  prometheus.yml: 
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
      - targets: ['localhost:9090']
```



```shell
[root@k8s-80 k8syaml]# vim prometheus-rbac.yaml
```

```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: monitor
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - services
  - endpoints
  - pods
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - "extensions"
  resources:
    - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - configmaps
  - nodes/metrics
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: monitor
```



```shell
[root@k8s-80 k8syaml]# vim prometheus.yaml
```

```yaml
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: monitor
  labels:
    app: prometheus
spec:
  selector:
    app: prometheus
  type: ClusterIP
  ports:
    - name: web
      port: 9090
      targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: monitor
  labels:
    app: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      serviceAccountName: prometheus
      nodeSelector:
        node-role.kubernetes.io/bus: "true"
      initContainers:
      - name: "change-permission-of-directory"
        image: busybox
        command: ["/bin/sh"]
        args: ["-c", "chown -R 65534:65534 /prometheus"]
        securityContext:
          privileged: true
        volumeMounts:
        - mountPath: "/etc/prometheus"
          name: config-volume
        - mountPath: "/prometheus"
          name: data
      containers:
      - image: prom/prometheus
        name: prometheus-v2.35.0
        args:
        - "--config.file=/etc/prometheus/prometheus.yml"
        - "--storage.tsdb.path=/prometheus"  # 指定tsdb数据路径
        - "--web.enable-lifecycle"  # 支持热更新，直接执行localhost:9090/-/reload立即生效
        - "--web.console.libraries=/usr/share/prometheus/console_libraries"
        - "--web.console.templates=/usr/share/prometheus/consoles"
        ports:
        - containerPort: 9090
          name: http
        volumeMounts:
        - mountPath: "/etc/prometheus"
          name: config-volume
        - mountPath: "/prometheus"
          name: data
        resources:
          requests:
            cpu: 100m
            memory: 512Mi
          limits:
            cpu: 100m
            memory: 512Mi
      volumes:
      - name: data
        hostPath:
          path: /data/prometheus/
      - configMap:
          name: prometheus-config
        name: config-volume
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: prometheus
  namespace: monitor
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
spec:
  ingressClassName: nginx
  rules:
  - host: www.lin.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
         service:
           name: prometheus
           port:
             number: 9090
```





```
kubectl apply -f prometheus-ns.yaml
kubectl apply -f prometheus-cm.yaml
kubectl apply -f prometheus-rbac.yaml
kubectl apply -f prometheus.yaml
```



```
[root@k8s-80 k8syaml]# kubectl get ns
NAME              STATUS   AGE
default           Active   11d
ingress-nginx     Active   11d
kube-node-lease   Active   11d
kube-public       Active   11d
kube-system       Active   11d
monitor           Active   16m


[root@k8s-80 k8syaml]# kubectl get po -n monitor
NAME                          READY   STATUS    RESTARTS   AGE
prometheus-5b4fd7679d-fmmq9   1/1     Running   0          15m
```



```
[root@k8s-80 k8syaml]# kubectl get svc -n monitor
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
prometheus   ClusterIP   10.111.26.173   <none>        9090/TCP   16m

[root@k8s-80 k8syaml]# curl http://10.111.26.173:9090/metrics   # 会有对应监控参数详细信息出来
```

![](https://figure-bed-1304788733.cos.ap-guangzhou.myqcloud.com/typora/202206061633584.png)



```
# 浏览器输入www.xiaosu.com，页面与之前学的一样点击tags
# 因为在使用 kubeadm 版安装的时候已经在Windows上做好解析，所以可以直接使用www.xiaosu.com
```

![](https://figure-bed-1304788733.cos.ap-guangzhou.myqcloud.com/typora/202206061638039.png)





```
[root@k8s-80 k8syaml]# vim prometheus-cm.yaml
```

```
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
      - targets: ['localhost:9090']
    - job_name: 'coredns'                  # 增加这一个job
      static_configs:
      - targets: ['10.96.0.10:9153']
```



```
[root@k8s-80 k8syaml]# kubectl apply -f prometheus-cm.yaml


[root@k8s-80 k8syaml]# kubectl get po -n monitor
NAME                          READY   STATUS    RESTARTS        AGE
prometheus-5b4fd7679d-fmmq9   1/1     Running   1 (3h46m ago)   9h


[root@k8s-80 k8syaml]# kubectl get svc -n monitor
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
prometheus   ClusterIP   10.111.26.173   <none>        9090/TCP   9h


[root@k8s-80 k8syaml]# curl -XPOST http://10.111.26.173:9090/-/reload


[root@k8s-80 k8syaml]# kubectl exec -it prometheus-5b4fd7679d-fmmq9 -n monitor -- cat /etc/prometheus/prometheus.yml
Defaulted container "prometheus" out of: prometheus, change-permission-of-directory (init)
global:
  scrape_interval: 15s
  evaluation_interval: 15s
scrape_configs:
- job_name: 'prometheus'
  static_configs:
  - targets: ['localhost:9090']
- job_name: 'coredns'
  static_configs:
  - targets: ['10.96.0.10:9153']
```



```
# 浏览器输入www.xiaosu.com，页面与之前学的一样点击tags，发现成功添加监控coredns
```

![](https://figure-bed-1304788733.cos.ap-guangzhou.myqcloud.com/typora/202206070029818.png)





```
# apiserver自身也提供了/metrics 的api来提供监控数据
# 使用里面的token，也会出现一堆详细监控参数

[root@k8s-80 k8syaml]# test=`kubectl exec -it prometheus-5b4fd7679d-fmmq9 -n monitor -- cat /var/run/secrets/kubernetes.io/serviceaccount/token`


[root@k8s-80 k8syaml]# curl -k  -H "Authorization: Bearer $test " https://192.168.188.80:6443/metrics
```

![](https://figure-bed-1304788733.cos.ap-guangzhou.myqcloud.com/typora/202206070034515.png)



```
[root@k8s-80 k8syaml]# vim prometheus-cm.yaml 
```

```
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitor
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
      - targets: ['localhost:9090']
    - job_name: 'coredns'
      static_configs:
      - targets: ['10.96.0.10:9153']
    - job_name: 'kubernetes-apiserver'   # 添加这一段job
      static_configs:
      - targets: ['10.96.0.1']
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
```

```
[root@k8s-80 k8syaml]# kubectl apply -f prometheus-cm.yaml


[root@k8s-80 k8syaml]# curl -XPOST http://10.111.26.173:9090/-/reload


[root@k8s-80 k8syaml]# kubectl exec -it prometheus-5b4fd7679d-fmmq9 -n monitor -- cat /etc/prometheus/prometheus.yml
Defaulted container "prometheus" out of: prometheus, change-permission-of-directory (init)
global:
  scrape_interval: 15s
  evaluation_interval: 15s
scrape_configs:
- job_name: 'prometheus'
  static_configs:
  - targets: ['localhost:9090']
- job_name: 'coredns'
  static_configs:
  - targets: ['10.96.0.10:9153']
- job_name: 'kubernetes-apiserver'
  static_configs:
  - targets: ['10.96.0.1']
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecure_skip_verify: true
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
```



```
# 浏览器输入www.xiaosu.com，页面与之前学的一样点击tags，发现成功添加监控kubernetes-apiserver
```

![](https://figure-bed-1304788733.cos.ap-guangzhou.myqcloud.com/typora/202206070038623.png)



```
[root@k8s-80 k8syaml]# kubectl get no
NAME     STATUS   ROLES                  AGE   VERSION
k8s-80   Ready    control-plane,master   11d   v1.22.3
k8s-81   Ready    bus,php                11d   v1.22.3
k8s-82   Ready    bus,go                 11d   v1.22.3


[root@k8s-80 k8syaml]# kubectl describe no k8s-81   # 里面的一些参数可当做环境变量来使用


[root@k8s-80 k8syaml]# kubectl get no k8s-81 -o yaml   # 同样里面的一些参数可当做环境变量来使用
# 比如以下查询出来的参数在 prometheus 对应的yaml文件中写成 HOSTIP
status:
  addresses:
  - address: 192.168.188.81
    type: InternalIP
  - address: k8s-81
    type: Hostname
```



```
[root@k8s-80 k8syaml]# vim prometheus-node.yaml
```

```yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: monitor
  labels:
    app: node-exporter
spec:
  selector:
    matchLabels:
      app: node-exporter
  template:
    metadata:
      labels:
        app: node-exporter
    spec:
      hostPID: true
      hostIPC: true
      hostNetwork: true
      nodeSelector:
        kubernetes.io/os: linux
      containers:
      - name: node-exporter
        image: prom/node-exporter:v1.0.1
        args:
        - --web.listen-address=$(HOSTIP):9100
        - --path.procfs=/host/proc
        - --path.sysfs=/host/sys
        - --path.rootfs=/host/root
        - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)
        - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$
        ports:
        - containerPort: 9100
        env:
        - name: HOSTIP
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        resources:
          requests:
            cpu: 150m
            memory: 180Mi
          limits:
            cpu: 150m
            memory: 180Mi
        securityContext:
          runAsNonRoot: true
          runAsUser: 65534
        volumeMounts:
        - name: proc
          mountPath: /host/proc
        - name: sys
          mountPath: /host/sys
        - name: root
          mountPath: /host/root
          mountPropagation: HostToContainer
          readOnly: true
      tolerations:
      - operator: "Exists"
      volumes:
      - name: proc
        hostPath:
          path: /proc
      - name: dev
        hostPath:
          path: /dev
      - name: sys
        hostPath:
          path: /sys
      - name: root
        hostPath:
          path: /
```

```
[root@k8s-80 k8syaml]# kubectl apply -f prometheus-node.yaml

[root@k8s-80 k8syaml]# kubectl get po -n monitor
NAME                          READY   STATUS    RESTARTS        AGE
node-exporter-9zrlj           1/1     Running   0               59s
node-exporter-v72fm           1/1     Running   0               59s
node-exporter-zc5zp           1/1     Running   0               59s
prometheus-5b4fd7679d-fmmq9   1/1     Running   1 (4h14m ago)   9h
```



```
[root@k8s-80 k8syaml]# vim prometheus-cm.yaml 
```

```yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitor
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
      - targets: ['localhost:9090']
    - job_name: 'coredns'
      static_configs:
      - targets: ['10.96.0.10:9153']
    - job_name: 'kubernetes-apiserver'
      static_configs:
      - targets: ['10.96.0.1']
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    - job_name: 'kubernetes-sd-node-exporter'   # 添加这段
      kubernetes_sd_configs:
        - role: node
```

```
[root@k8s-80 k8syaml]# kubectl apply -f prometheus-cm.yaml

[root@k8s-80 k8syaml]# curl -XPOST http://10.111.26.173:9090/-/reload
```

```
[root@k8s-80 k8syaml]# kubectl exec -it prometheus-5b4fd7679d-fmmq9 -n monitor -- cat /etc/prometheus/prometheus.yml
Defaulted container "prometheus" out of: prometheus, change-permission-of-directory (init)
global:
  scrape_interval: 15s
  evaluation_interval: 15s
scrape_configs:
- job_name: 'prometheus'
  static_configs:
  - targets: ['localhost:9090']
- job_name: 'coredns'
  static_configs:
  - targets: ['10.96.0.10:9153']
- job_name: 'kubernetes-apiserver'
  static_configs:
  - targets: ['10.96.0.1']
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecure_skip_verify: true
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
- job_name: 'kubernetes-sd-node-exporter'
  kubernetes_sd_configs:
    - role: node
```





![](https://figure-bed-1304788733.cos.ap-guangzhou.myqcloud.com/typora/202206070050859.png)



```
[root@k8s-80 k8syaml]# netstat -aulntp | grep 10250
tcp6       0      0 :::10250                :::*                    LISTEN      1030/kubelet        
tcp6       0      0 192.168.188.80:10250    192.168.188.82:19937    TIME_WAIT   -                   
tcp6       0      0 192.168.188.80:10250    192.168.188.82:49223    TIME_WAIT   -                   
tcp6       0      0 192.168.188.80:10250    192.168.188.82:11174    TIME_WAIT   -                   
tcp6       0      0 192.168.188.80:10250    192.168.188.82:55415    TIME_WAIT   -                   
tcp6       0      0 192.168.188.80:10250    192.168.188.82:55290    ESTABLISHED 1030/kubelet
```

```
[root@k8s-80 k8syaml]# vim prometheus-cm.yaml
```

```
# 增加以下重新打标
- job_name: 'kubernetes-sd-node-exporter'   # 这段已经写了
      kubernetes_sd_configs:
        - role: node
      relabel_configs:   # 与上面role同级
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:9100'
        target_label: __address__
        action: replace
```

```
[root@k8s-80 k8syaml]# kubectl apply -f prometheus-cm.yaml

[root@k8s-80 k8syaml]# curl -XPOST http://10.111.26.173:9090/-/reload

[root@k8s-80 k8syaml]# kubectl exec -it prometheus-5b4fd7679d-fmmq9 -n monitor -- cat /etc/prometheus/prometheus.yml
```



![](https://figure-bed-1304788733.cos.ap-guangzhou.myqcloud.com/typora/202206070109442.png)