Kubernetes成本计量监控

git clone https://github.com/prometheus-operator/kube-prometheus.git

kubectl apply --server-side -f manifests/setup
kubectl wait \
	--for condition=Established \
	--all CustomResourceDefinition \
	--namespace=monitoring

// 删除NetworkPolicy YAML文件
-rw-r--r-- 1 root123 root123  977 Jan 17 02:15 alertmanager-networkPolicy.yaml
-rw-r--r-- 1 root123 root123  722 Jan 17 02:15 blackboxExporter-networkPolicy.yaml
-rw-r--r-- 1 root123 root123  652 Jan 17 02:15 grafana-networkPolicy.yaml
-rw-r--r-- 1 root123 root123  724 Jan 17 02:15 kubeStateMetrics-networkPolicy.yaml
-rw-r--r-- 1 root123 root123  671 Jan 17 02:15 nodeExporter-networkPolicy.yaml
-rw-r--r-- 1 root123 root123  565 Jan 17 02:16 prometheusAdapter-networkPolicy.yaml
-rw-r--r-- 1 root123 root123 1072 Jan 17 02:16 prometheus-networkPolicy.yaml
-rw-r--r-- 1 root123 root123  694 Jan 17 02:16 prometheusOperator-networkPolicy.yaml


// 修改docker.io镜像仓库源
grafana-deployment.yaml:        image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/grafana/grafana:11.4.0
kubeStateMetrics-deployment.yaml:        image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0
prometheusAdapter-deployment.yaml:        image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.12.0

kubectl apply -f manifests/

helm pull opencost/opencost

tar xf opencost-1.43.1.tgz
cd opencost/
vi values.yaml
service:
  enabled: true
  # --  Annotations to add to the service
  annotations: {}
  # --  Labels to add to the service account
  labels: {}
  # --  Kubernetes Service type
  type: NodePort
  # -- NodePort if service type is NodePort
opencost:
  # -- <SECRET_NAME> for the secret containing the Cloud Costs cloud-integration.json https://www.opencost.io/docs/configuration/#cloud-costs
  # -- kubectl create secret generic <SECRET_NAME> --from-file=cloud-integration.json -n opencost
  cloudIntegrationSecret: ""
  exporter:
    # API_PORT for the cost-model to listen on
    apiPort: 9003
    # debugPort: 40000 # for development purposes (debugging with delve) and not for production.
    # -- The GCP Pricing API requires a key. This is supplied just for evaluation.
    cloudProviderApiKey: ""
    # -- Default cluster ID to use if cluster_id is not set in Prometheus metrics.
    defaultClusterId: 'sukai-cluster'
  
  customPricing:
    # -- Enables custom pricing configuration
    enabled: true
    # -- Customize the configmap name used for custom pricing
    configmapName: custom-pricing-model
    # -- Path for the pricing configuration.
    configPath: /tmp/custom-config
    # -- Configures the pricing model provided in the values file.
    createConfigmap: true
    # -- Sets the provider type for the custom pricing file.
    provider: custom
    # -- More information about these values here: https://www.opencost.io/docs/configuration/on-prem#custom-pricing-using-the-opencost-helm-chart
    costModel:
      description: "Default prices based on GCP us-central1"
      CPU: 0.031611
      spotCPU: 0.006655
      RAM: 0.004237
      spotRAM: 0.000892
      GPU: 0.95
      storage: 0.00005479452
      zoneNetworkEgress: 0.01
      regionNetworkEgress: 0.01
      internetNetworkEgress: 0.12
  prometheus:
    # -- Secret name that contains credentials for Prometheus
    secret_name: ~
    # -- Existing secret name that contains credentials for Prometheus
    existingSecretName: ~
    # -- Prometheus Basic auth username
    username: ""
    # -- Key in the secret that references the username
    username_key: DB_BASIC_AUTH_USERNAME
    # -- Prometheus Basic auth password
    password: ""
    # -- Key in the secret that references the password
    password_key: DB_BASIC_AUTH_PW
    # -- Prometheus Bearer token
    bearer_token: ""
    bearer_token_key: DB_BEARER_TOKEN
    # -- If true, opencost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift
    kubeRBACProxy: false
    # OPTIONAL. The following configs only to be enabled when using a Prometheus instance already installed in the cluster.
    # -- If true, the helm chart will create a ClusterRoleBinding to grant the OpenCost ServiceAccount access to query Prometheus.
    createMonitoringClusterRoleBinding: false
    # -- If true, create a Role and RoleBinding to allow Prometheus to list and watch OpenCost resources.
    createMonitoringResourceReaderRoleBinding: false
    # -- Name of the Prometheus serviceaccount to bind to the Resource Reader Role Binding.
    monitoringServiceAccountName: prometheus-k8s
    # -- Namespace of the Prometheus serviceaccount to bind to the Resource Reader Role Binding.
    monitoringServiceAccountNamespace: monitoring
    external:
      # -- Use external Prometheus (eg. Grafana Cloud)
      enabled: false
      # -- External Prometheus url
      url: "https://prometheus.example.com/prometheus"
    internal:
      # -- Use in-cluster Prometheus
      enabled: true
      # -- Service name of in-cluster Prometheus
      serviceName: prometheus-k8s
      # -- Namespace of in-cluster Prometheus
      namespaceName: monitoring

kubectl create ns opencost
helm install opencost opencost --namespace opencost
  

vi prometheus-opencost.yaml
- job_name: opencost
  honor_labels: true
  scrape_interval: 1m
  scrape_timeout: 10s
  metrics_path: /metrics
  scheme: http
  static_configs:
  - targets:
    - opencost.opencost.svc:9003

kubectl create secret generic additional-scrape-configs --from-file=prometheus-opencost.yaml --dry-run=client -oyaml > opencost-scrape-configs.yaml

more opencost-scrape-configs.yaml
apiVersion: v1
data:
  prometheus-opencost.yaml: LSBqb2JfbmFtZTogb3BlbmNvc3QKICBob25vcl9sYWJlbHM6IHRydWUKICBzY3JhcGVfaW50ZXJ2YWw6IDFtCiAgc2NyYXBlX3RpbWVvdX
Q6IDEwcwogIG1ldHJpY3NfcGF0aDogL21ldHJpY3MKICBzY2hlbWU6IGh0dHAKICBzdGF0aWNfY29uZmlnczoKICAtIHRhcmdldHM6CiAgICAtIG9wZW5jb3N0Lm9wZW5jb3N0
LnN2Yzo5MDAzCg==
kind: Secret
metadata:
  creationTimestamp: null
  name: additional-scrape-configs

kubectl apply -f opencost-scrape-configs.yaml -n monitoring


vi prometheus-prometheus.yaml
  // 配置Prometheus抓取任务
  additionalScrapeConfigs:
    name: additional-scrape-configs
    key: prometheus-opencost.yaml


cat prometheus-prometheus.yaml
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  labels:
    app.kubernetes.io/component: prometheus
    app.kubernetes.io/instance: k8s
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 3.1.0
  name: k8s
  namespace: monitoring
spec:
  alerting:
    alertmanagers:
    - apiVersion: v2
      name: alertmanager-main
      namespace: monitoring
      port: web
  enableFeatures: []
  externalLabels: {}
  image: quay.io/prometheus/prometheus:v3.1.0
  nodeSelector:
    kubernetes.io/os: linux
  podMetadata:
    labels:
      app.kubernetes.io/component: prometheus
      app.kubernetes.io/instance: k8s
      app.kubernetes.io/name: prometheus
      app.kubernetes.io/part-of: kube-prometheus
      app.kubernetes.io/version: 3.1.0
  podMonitorNamespaceSelector: {}
  podMonitorSelector: {}
  probeNamespaceSelector: {}
  probeSelector: {}
  replicas: 2
  resources:
    requests:
      memory: 400Mi
  ruleNamespaceSelector: {}
  ruleSelector: {}
  scrapeConfigNamespaceSelector: {}
  scrapeConfigSelector: {}
  securityContext:
    fsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: prometheus-k8s
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  additionalScrapeConfigs:
    name: additional-scrape-configs
    key: prometheus-opencost.yaml
  version: 3.1.0

kubectl apply -f prometheus-prometheus.yaml