监控k8s

抓取apiserver的监控指标

     - job_name: 'kubernetes-apiservers'
        kubernetes_sd_configs:
          - role: endpoints

        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_service_name,__meta_kubernetes_endpoint_port_name]
            # 把符合要求的保留下来
            action: keep
            regex: default;kubernetes;https

抓取kube-controller-manager的监控指标

        # 1、创建svc，标签选择选中，
        apiVersion: v1
        kind: Service
        metadata:
          labels:
            app.kubernetes.io/component: kube-controller-manager
            app.kubernetes.io/name: kube-controller-manager
            k8s-app: kube-controller-manager
          name: kube-controller-manager
          namespace: kube-system
        spec:
          clusterIP: None
          ports:
          - name: https-metrics
            port: 10257
            targetPort: 10257
            protocol: TCP
          selector:
            component: kube-controller-manager


        # 2、你可以查看下上述svc其关联的endpoint用的地址都是物理机的地址，而每台机器上的controller-manager都监听在127.0.0.1，所以是无法访问的，需要修改其默认监听
        vi /etc/kubernetes/manifests/kube-controller-manager.yaml # 修改--bind-address=127.0.0.1

        每台master节点都改


        # 3、添加监控

              - job_name: 'kube-controller-manager'
                kubernetes_sd_configs:
                - role: endpoints
                scheme: https
                tls_config:
                  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                  insecure_skip_verify: true
                bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
                relabel_configs:
                - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_service_name,__meta_kubernetes_endpoint_port_name]
                  action: keep
                  regex: kube-system;kube-controller-manager;https-metrics
                  
                  
                  # 这里的https-metrics名字与你svc中为ports起的名字保持一致
            
    
    抓取kube-scheduler的监控指标
        同上
        
    抓取etcd的监控指标
        # 1、修改每个master节点上etcd的静态配置yaml
        - --listen-metrics-urls=http://127.0.0.1:2381


        # 2、etcd-service.yaml
        apiVersion: v1
        kind: Service
        metadata:
          namespace: kube-system
          name: etcd
          labels:
            k8s-app: etcd
        spec:
          selector:
            component: etcd
          type: ClusterIP
          clusterIP: None
          ports:
            - name: http
              port: 2381
              targetPort: 2381
              protocol: TCP

        # 3、etcd监控
        - job_name: 'etcd'
          kubernetes_sd_configs:
          - role: endpoints
          scheme: http
          
          relabel_configs:
          - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
            action: keep
            regex: kube-system;etcd;http

自动发现业务服务

      - job_name: 'kubernetes-endpoints'
        kubernetes_sd_configs:
          - role: endpoints
        relabel_configs:
          # 1、匹配元数据__meta_kubernetes_service_annotation_prometheus_io_scrape包含"true"的留下
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
            action: keep
            regex: true
            
          # 2、__meta_kubernetes_service_annotation_prometheus_io_scheme的值中包含0或1个https，则改名为__scheme__
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
            action: replace
            target_label: __scheme__
            regex: (https?)
            
          # 3、__meta_kubernetes_service_annotation_prometheus_io_path的值至少有一个任意字符，则改名为__metrics_path__
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
            action: replace
            target_label: __metrics_path__
            regex: (.+)
            
          # 4、提取ip与port拼接到一起格式为: 1.1.1.1:3333，然后赋值给新label名：__address__
          - source_labels:
              [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
            action: replace
            target_label: __address__
            regex: ([^:]+)(?::\d+)?;(\d+) # RE2 正则规则，+是一次多多次，?是0次或1次，其中?:表示非匹>配组(意思就是不获取匹配结果)
            replacement: $1:$2
            
          # 5、添加标签
          - action: labelmap
            regex: __meta_kubernetes_service_label_(.+)
            
          # 6、名称空间标签替换为kubernetes_namespace
          - source_labels: [__meta_kubernetes_namespace]
            action: replace
            target_label: kubernetes_namespace
            
          # 7、服务名替换为kubernetes_name
          - source_labels: [__meta_kubernetes_service_name]
            action: replace
            target_label: kubernetes_name
          
          # 8、pod名替换为kubernetes_pod_name
          - source_labels: [__meta_kubernetes_pod_name]
            action: replace
            target_label: kubernetes_pod_name

自动发现测试

            # prome-redis.yaml
            apiVersion: apps/v1
            kind: Deployment
            metadata:
              name: redis
              namespace: monitor
            spec:
              selector:
                matchLabels:
                  app: redis
              template:
                metadata:
                  labels:
                    app: redis
                spec:
                  containers:
                    - name: redis
                      image: redis:4
                      resources:
                        requests:
                          cpu: 100m
                          memory: 100Mi
                      ports:
                        - containerPort: 6379
                    - name: redis-exporter
                      image: oliver006/redis_exporter:latest
                      resources:
                        requests:
                          cpu: 100m
                          memory: 100Mi
                      ports:
                        - containerPort: 9121
            ---
            kind: Service
            apiVersion: v1
            metadata:
              name: redis
              namespace: monitor
              annotations: # --------------------------------》  添加
                prometheus.io/scrape: 'true'
                prometheus.io/port: '9121'
            spec:
              selector:
                app: redis
              ports:
                - name: redis
                  port: 6379
                  targetPort: 6379
                - name: prom
                  port: 9121
                  targetPort: 9121