实现一个MutatingAdmissionWebhook

1

kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger ...

1

kube-apiserver --disable-admission-plugins=PodNodeSelector,AlwaysDeny ...

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# 根证书CA私钥
openssl genrsa -out rootCA.key 4096
# 根证书
openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 3650 -out rootCA.crt -subj "/CN=BaiqiKubernetes"
# SAN配置文件
echo '[req]
      default_bits = 2048
      prompt = no
      default_md = sha256
      distinguished_name = dn
      req_extensions = req_ext

      [dn]
      CN = baiqi.io

      [req_ext]
      subjectAltName = @alt_names

      [alt_names]
      DNS.1 = *.default.svc
      DNS.2 = *.kube-system.svc
      DNS.3 = *.ops.svc
      DNS.4 = *.baiqi.io
      DNS.5 = *.ops.baiqi.io' > server.cnf

# 证书私钥
openssl genrsa -out server.key 2048
# 根据配置和私钥生成证书签名请求文件CSR
openssl req -new -key server.key -out server.csr -config server.cnf
# 使用根证书签发证书
openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out server.crt -days 1095 -extensions req_ext -extfile server.cnf

1
2
3
4
5
6
7

-rw-rw-r-- rootCA.crt
-rw------- rootCA.key
-rw-rw-r-- rootCA.srl
-rw-rw-r-- server.cnf
-rw-rw-r-- server.crt
-rw-rw-r-- server.csr
-rw------- server.key

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193

package main

import (
    "crypto/tls"
    "encoding/json"
    "flag"
    "fmt"
    "io"
    "log"
    "log/slog"
    "net/http"
    "os"
    "strconv"
    "strings"

    // api group/version类型定义
    admissionV1 "k8s.io/api/admission/v1"
    v1 "k8s.io/api/core/v1"
    // Kubernetes 对象的类型定义、注册和管理，以及提供通用的对象处理功能
    "k8s.io/apimachinery/pkg/runtime"
    // 对象的序列化和反序列化操作，提供了多种序列化器的实现和操作方法
    "k8s.io/apimachinery/pkg/runtime/serializer"
)

var (
    port        int
    tlsKey      string
    tlsCert     string
    imageSource string // 以逗号分割的仓库地址，默认改为第一个仓库地址
)

var logger *slog.Logger

// patch 操作定义
type PatchOperation struct {
    Op    string      `json:"op"`
    Path  string      `json:"path"`
    Value interface{} `json:"value,omitempty"`
}

func init() {
    flag.IntVar(&port, "port", 9099, "Admisson controller port")
    flag.StringVar(&tlsKey, "tls-key", "/app/certs/tls.key", "Private key for TLS")
    flag.StringVar(&tlsCert, "tls-crt", "/app/certs/tls.crt", "TLS certificate")
    flag.StringVar(&imageSource, "image-url", "", "image url host")
}

func main() {
    flag.Parse()

    logHandler := slog.NewJSONHandler(
        os.Stdout, &slog.HandlerOptions{
            Level: slog.LevelInfo,
        },
    )
    logger = slog.New(logHandler)
    logger.With("mutateAdmissionName", "image-admission")

    // 加载证书和私钥
    certs, err := tls.LoadX509KeyPair(tlsCert, tlsKey)
    if err != nil {
        panic(err)
    }

    // webhook 路由端点
    http.HandleFunc("/pod-mutate", serverPod)

    logger.Info("Starting server...", "port", port)
    server := http.Server{
        Addr: fmt.Sprintf(":%d", port),
        TLSConfig: &tls.Config{
            Certificates: []tls.Certificate{certs},
        },
    }

    // 启动TLS监听
    if err := server.ListenAndServeTLS("", ""); err != nil {
        log.Panic(err)
    }
}

// imagePatches 按要求生成patch的操作列表，即前面说的需求1
func imagePatches(pod *v1.Pod) []PatchOperation {
    var patches []PatchOperation
    var image string
    patch := PatchOperation{
        Op: "replace",
    }

    imageSourceList := strings.Split(imageSource, ",")
    for i, container := range pod.Spec.Containers {
        imagePair := strings.Split(container.Image, ":")
        if strings.Contains(imagePair[0], ".") {
            image = strings.Replace(container.Image, strings.Split(imagePair[0], "/")[0], imageSourceList[0], -1)
            patch.Path = "/spec/containers/" + strconv.Itoa(i) + "/image"
            patch.Value = image
            patches = append(patches, patch)
            continue
        }
        image = imageSourceList[0] + "/" + container.Image
        patch.Path = "/spec/containers/" + strconv.Itoa(i) + "/image"
        patch.Value = image
        patches = append(patches, patch)
    }
    return patches
}

// parseAdmissionReview 解析准入请求
func parseAdmissionReview(r *http.Request, decoder runtime.Decoder) (*admissionV1.AdmissionReview, error) {
    var admissionReviewRequest = &admissionV1.AdmissionReview{}
    body, err := io.ReadAll(r.Body)
    defer r.Body.Close()
    if err != nil {
        return nil, err
    }
    // err = json.Unmarshal(body, &admissionReviewRequest)
    _, _, err = decoder.Decode(body, nil, admissionReviewRequest)
    if err != nil {
        return nil, err
    }
    return admissionReviewRequest, nil
}

// http handler，处理请求
func serverPod(w http.ResponseWriter, r *http.Request) {

    // 创建反序列化器
    scheme := runtime.NewScheme()
    codecFactory := serializer.NewCodecFactory(scheme)
    deserializer := codecFactory.UniversalDeserializer()

    admissionReviewRequest, err := parseAdmissionReview(r, deserializer)
    if err != nil {
        http.Error(w, err.Error(), http.StatusBadRequest)
        logger.Error("unable to parse AdmissionReview,", "err", err)
        return
    }
    // 只针对pods资源
    if admissionReviewRequest.Request.Kind.Kind != "Pod" {
        http.Error(w, "admissionReviewRequest.Request.Kind.Kind != Pod", http.StatusForbidden)
        logger.Error("admission review requires Kind Pod, Please check register config", "err", err)
        return
    }
    logger.Debug("admission request", admissionReviewRequest)

    // 从请求中解析出pod声明
    pod := &v1.Pod{}
    _, _, err = deserializer.Decode(admissionReviewRequest.Request.Object.Raw, nil, pod)
    if err != nil {
        http.Error(w, err.Error(), http.StatusBadRequest)
        logger.Error("unable to decode AdmissionReview object to Pod", "err", err)
        return
    }
    logger.Debug("requested object", pod.Spec.Containers)

    // 生成json patch列表
    patches := imagePatches(pod)

    logger.Debug("patches", patches)

    patchesBytes, err := json.Marshal(patches)
    if err != nil {
        http.Error(w, err.Error(), http.StatusInternalServerError)
        logger.Error("unable to encode patch", "err", err)
        return
    }

    // 组装响应
    admissionResponse := &admissionV1.AdmissionResponse{}
    // 响应中的UID必须和请求中的一致
    admissionResponse.UID = admissionReviewRequest.Request.UID
    // 允许准入
    admissionResponse.Allowed = true
    admissionResponse.PatchType = func() *admissionV1.PatchType { pt := admissionV1.PatchTypeJSONPatch; return &pt }()
    admissionResponse.Patch = patchesBytes
    admissionResponse.Warnings = nil

    var admissionReviewResponse admissionV1.AdmissionReview
    admissionReviewResponse.Response = admissionResponse
    admissionReviewResponse.SetGroupVersionKind(admissionReviewRequest.GroupVersionKind())

    responseBytes, err := json.Marshal(admissionReviewResponse)
    if err != nil {
        logger.Error("unable to encode AdmissionReview response", "err", err)
        http.Error(w, err.Error(), http.StatusInternalServerError)
        return
    }

    // 准入要求的头和状态码
    w.Header().Set("Content-Type", "application/json")
    w.WriteHeader(http.StatusOK)
    _, _ = w.Write(responseBytes)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

FROM golang:1.24 AS builder

ENV GO111MODULE=on
ENV GOPROXY=https://goproxy.cn,direct
WORKDIR /opt/
COPY . .
RUN go mod download && go mod verify
RUN GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -v -o image-admission

FROM alpine:3.18 AS prod
WORKDIR /opt/
COPY --from=builder /opt/image-admission ./image-admission
ENTRYPOINT ["/opt/image-admission"]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: image-admission
  namespace: kube-system
  labels:
    app: image-admission
spec:
  replicas: 1
  selector:
    matchLabels:
      app: image-admission
  template:
    metadata:
      name: image-admission
      labels:
        app: image-admission
    spec:
      containers:
        - name: image-admission
          command:
            - /opt/image-admission
            - "-image-url"
            - "a.b.c,x.y.z"
            - "-port"
            - "9099"
          image: image-admission:v0.0.1
          imagePullPolicy: Always
          ports:
            - containerPort: 9099
              protocol: TCP
          volumeMounts:
            - mountPath: "/app/certs"
              name: tls
      restartPolicy: Always
      imagePullSecrets:
        - name: hub-in
      volumes:
        - name: tls
          secret:
            secretName: image-admission-tls

---
apiVersion: v1
kind: Secret
metadata:
  name: image-admission-tls
  namespace: kube-system
  labels:
    app: image-admission
type: kubernetes.io/tls
data:
  tls.crt: xx
  tls.key: yy

---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: image-admission
  name: image-admission
  namespace: kube-system
spec:
  internalTrafficPolicy: Cluster
  ipFamilies:
    - IPv4
  ipFamilyPolicy: SingleStack
  ports:
    - name: https
      port: 443
      protocol: TCP
      targetPort: 9099
  selector:
    app: image-admission
  sessionAffinity: None
  type: ClusterIP

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  labels:
    app: image-admission
  name: image-admission
  namespace: kube-system
webhooks:
  - name: image-admission.x.z
    timeoutSeconds: 3
    failurePolicy: Fail
    clientConfig:
      caBundle: cacaca=
      service:
        namespace: kube-system
        name: image-admission
        path: /pod-mutate
        port: 443
    namespaceSelector:
      matchExpressions:
        - key: kubernetes.io/metadata.name
          operator: In
          values: ["default", "x"]
    rules:
      - operations: ["CREATE", "UPDATE"]
        # "" 是core组
        apiGroups: [""]
        apiVersions: ["v1"]
        resources: ["pods"]
        scope: "Namespaced"
    admissionReviewVersions:
      - v1
    sideEffects: None

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

apiVersion: v1
kind: Pod
metadata:
  name: nginx
spec:
  containers:
  - name: nginx
    image: nginx:1.14.2
    ports:
    - containerPort: 80

1
2

$ kubectl get pod nginx -o jsonpath='{.spec.containers[0].image}'
a.b.c/nginx:1.14.2

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

apiVersion: audit.k8s.io/v1
kind: Policy
# 忽略健康检查请求
omitStages:
  - "RequestReceived"
rules:
  # 记录pod变化
  - level: RequestResponse
    resources:
      - group: ""
        resources: ["pods"]
  # 对所有其他请求进行元数据级别的审计
  - level: Metadata
    resources:
      - group: ""

1
2
3
4
5
6

kube-apiserver \
  --audit-policy-file=/path/to/audit-policy.yaml \
  --audit-log-path=/var/log/kubernetes/audit.log \
  --audit-log-maxsize=100 \
  --audit-log-maxbackup=10 \
  --audit-log-maxage=30