# 纸面命令 —— 在实验集群执行，勿对 116 乱打 taint
kubectl label node gpu-w01 nvidia.com/gpu.present=true --overwrite
kubectl label node gpu-w01 nvidia.com/gpu.product=NVIDIA-L40S --overwrite
kubectl taint nodes gpu-w01 nvidia.com/gpu=true:NoSchedule

kubectl get nodes -o wide
# 期望：3× control-plane Ready；6× worker Ready

kubectl get pods -A | grep -E 'calico|cilium|flannel|kube-system'
# CNI / dns 无 CrashLoop

kubectl get cs   # 老版本；新版本看 component status 等价检查

# 纸面 —— 版本号以 NVIDIA 文档稳定版为准
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update
helm upgrade --install gpu-operator nvidia/gpu-operator \
  -n gpu-operator --create-namespace \
  -f patches_cursor/基础设施_K8s_GPU_单私有化集群完整案例/07-gpu-operator-values.yaml

kubectl get pods -n gpu-operator
# 期望各节点：driver（若启用）/ toolkit / device-plugin / gfd / dcgm-exporter Running

kubectl describe node gpu-w01 | findstr /C:"nvidia.com/gpu"
# Linux: kubectl describe node gpu-w01 | grep -A8 Capacity

nvidia.com/gpu: 2

apiVersion: batch/v1
kind: Job
metadata:
  name: gpu-smoke
  namespace: gpu-infer
spec:
  template:
    spec:
      restartPolicy: Never
      nodeSelector:
        nvidia.com/gpu.present: "true"
      tolerations:
        - key: nvidia.com/gpu
          operator: Equal
          value: "true"
          effect: NoSchedule
      containers:
        - name: cuda-smoke
          image: harbor.example.local/ai/cuda-sample:vectorAdd
          resources:
            limits:
              nvidia.com/gpu: "1"

kubectl apply -f 01-namespaces.yaml
kubectl apply -f - <<'EOF'
# …上列 Job…
EOF
kubectl logs job/gpu-smoke -n gpu-infer
kubectl delete job gpu-smoke -n gpu-infer

kubectl -n gpu-infer get deploy,po,svc,ingress
kubectl -n gpu-infer rollout status deploy/vllm-qwen36-27b

# 集群内探活（示例）
kubectl -n gpu-infer run curl --rm -it --image=curlimages/curl -- \
  curl -sS http://vllm-qwen36-27b:8000/health

curl -sS http://vllm-qwen.gpu-k8s.example.local/health
curl -sS http://vllm-qwen.gpu-k8s.example.local/v1/models

开发改 manifests / CI 出镜像
        │
        ▼
  Harbor  ←  push ai/vllm-openai:v0.8.5
        │
  GitLab  ←  overlays/prod 更新 image tag
        │
  Argo CD ←  sync Application gpu-infer-workloads
        │
  gpu-infer NS 滚动更新 vLLM

kubectl apply -f patches_cursor/基础设施_K8s_GPU_单私有化集群完整案例/05-argo-application.yaml
argocd app get gpu-infer-workloads
# 或
kubectl -n argocd get application gpu-infer-workloads

gpu-infer-manifests/
  base/           # Deployment / Service 公共
  overlays/prod/  # 副本数、tag、Ingress host
  overlays/lab/

Dify → LiteLLM(207:4000) model=qwen3.6-27b
         │ 主
         └─► http://vllm-qwen.gpu-k8s.example.local/v1
         │ 备（fallback）
         └─► http://116.62.10.183:8000/v1   # 现网 Docker，作回滚锚点

# 在 207 或开发机
curl -sS http://192.168.110.207:4000/v1/chat/completions \
  -H "Authorization: Bearer sk-litellm-bocheng-207" \
  -H "Content-Type: application/json" \
  -d "{\"model\":\"qwen3.6-27b\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":16}"

# 1）路由回滚：api_base 改回 http://116.62.10.183:8000/v1，去掉或倒置 fallback
# 2）restart litellm，curl 验收
# 3）若 K8s 侧坏了：Argo 回滚 Application 或 git revert overlays/prod

现在：  Dify/LiteLLM@207  →  Docker vLLM@116
目标：  Dify/LiteLLM@207  →  Ingress→vLLM@单私有化 GPU K8s（116 作 fallback）
理论：  见姊妹篇；多集群/公网分配仍读理论篇，不在本案例展开

# 案例集群命名空间（纸面跟读，勿在 207/116 执行）
apiVersion: v1
kind: Namespace
metadata:
  name: gpu-operator
---
apiVersion: v1
kind: Namespace
metadata:
  name: gpu-infer
  labels:
    purpose: inference
---
apiVersion: v1
kind: Namespace
metadata:
  name: argocd

# gpu-infer 命名空间配额：最多 8 张卡（案例约定）
apiVersion: v1
kind: ResourceQuota
metadata:
  name: gpu-infer-quota
  namespace: gpu-infer
spec:
  hard:
    requests.nvidia.com/gpu: "8"
    limits.nvidia.com/gpu: "8"
    requests.cpu: "64"
    requests.memory: 256Gi
    pods: "40"
---
# GPU 节点污点：只跑推理（控制面 / 普通业务需容忍或避开）
# 实际执行：kubectl taint nodes gpu-w01 nvidia.com/gpu=true:NoSchedule
# 下列为文档对照用示意；污点通常用 kubectl / 节点 bootstrap，不靠本文件 apply
apiVersion: v1
kind: ConfigMap
metadata:
  name: gpu-node-ops-notes
  namespace: gpu-infer
data:
  taint: "nvidia.com/gpu=true:NoSchedule"
  label_present: "nvidia.com/gpu.present=true"
  label_product: "nvidia.com/gpu.product=NVIDIA-L40S"

# 主模型：qwen3.6-27b（对齐现网逻辑名）；1 卡；挂载模型 PVC
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-qwen36-27b
  namespace: gpu-infer
  labels:
    app: vllm-qwen36-27b
    model: qwen3.6-27b
spec:
  replicas: 2
  selector:
    matchLabels:
      app: vllm-qwen36-27b
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 1
  template:
    metadata:
      labels:
        app: vllm-qwen36-27b
        model: qwen3.6-27b
    spec:
      nodeSelector:
        nvidia.com/gpu.present: "true"
      tolerations:
        - key: nvidia.com/gpu
          operator: Equal
          value: "true"
          effect: NoSchedule
      containers:
        - name: vllm
          image: harbor.example.local/ai/vllm-openai:v0.8.5
          args:
            - --model
            - /models/qwen3.6-27b
            - --served-model-name
            - qwen3.6-27b
            - --host
            - 0.0.0.0
            - --port
            - "8000"
            - --tensor-parallel-size
            - "1"
          ports:
            - name: http
              containerPort: 8000
          env:
            - name: HUGGING_FACE_HUB_TOKEN
              valueFrom:
                secretKeyRef:
                  name: hf-token
                  key: token
                  optional: true
          resources:
            requests:
              cpu: "8"
              memory: 48Gi
              nvidia.com/gpu: "1"
            limits:
              cpu: "16"
              memory: 64Gi
              nvidia.com/gpu: "1"
          readinessProbe:
            httpGet:
              path: /health
              port: http
            initialDelaySeconds: 60
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /health
              port: http
            initialDelaySeconds: 120
            periodSeconds: 20
          volumeMounts:
            - name: model
              mountPath: /models/qwen3.6-27b
              readOnly: true
      volumes:
        - name: model
          persistentVolumeClaim:
            claimName: pvc-qwen36-27b
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-qwen36-27b
  namespace: gpu-infer
spec:
  accessModes: ["ReadOnlyMany"]
  resources:
    requests:
      storage: 80Gi
  storageClassName: nfs-models

# 备模 / 小模型占位：演示多 Deployment + 路由分流
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-qwen-small
  namespace: gpu-infer
  labels:
    app: vllm-qwen-small
    model: qwen-small
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm-qwen-small
  template:
    metadata:
      labels:
        app: vllm-qwen-small
        model: qwen-small
    spec:
      nodeSelector:
        nvidia.com/gpu.present: "true"
      tolerations:
        - key: nvidia.com/gpu
          operator: Equal
          value: "true"
          effect: NoSchedule
      containers:
        - name: vllm
          image: harbor.example.local/ai/vllm-openai:v0.8.5
          args:
            - --model
            - /models/qwen-small
            - --served-model-name
            - qwen-small
            - --host
            - 0.0.0.0
            - --port
            - "8000"
          ports:
            - name: http
              containerPort: 8000
          resources:
            requests:
              cpu: "4"
              memory: 24Gi
              nvidia.com/gpu: "1"
            limits:
              nvidia.com/gpu: "1"
              memory: 32Gi
          readinessProbe:
            httpGet:
              path: /health
              port: http
            initialDelaySeconds: 45
            periodSeconds: 10
          volumeMounts:
            - name: model
              mountPath: /models/qwen-small
              readOnly: true
      volumes:
        - name: model
          persistentVolumeClaim:
            claimName: pvc-qwen-small
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-qwen-small
  namespace: gpu-infer
spec:
  accessModes: ["ReadOnlyMany"]
  resources:
    requests:
      storage: 40Gi
  storageClassName: nfs-models
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-qwen36-27b
  namespace: gpu-infer
spec:
  selector:
    app: vllm-qwen36-27b
  ports:
    - name: http
      port: 8000
      targetPort: http
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-qwen-small
  namespace: gpu-infer
spec:
  selector:
    app: vllm-qwen-small
  ports:
    - name: http
      port: 8000
      targetPort: http
---
# 对 207 LiteLLM 暴露：内网 Ingress（示例域名）
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: vllm-ingress
  namespace: gpu-infer
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: "32m"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
spec:
  ingressClassName: nginx
  rules:
    - host: vllm-qwen.gpu-k8s.example.local
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: vllm-qwen36-27b
                port:
                  number: 8000
    - host: vllm-small.gpu-k8s.example.local
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: vllm-qwen-small
                port:
                  number: 8000

# Argo CD Application：以 Git 为准同步到「单」私有化 GPU 集群
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: gpu-infer-workloads
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://gitlab.example.local/ai-platform/gpu-infer-manifests.git
    targetRevision: main
    path: overlays/prod
  destination:
    server: https://kubernetes.default.svc
    namespace: gpu-infer
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
  # 生产可改为手动 sync；案例演示 automated + selfHeal

# 207 LiteLLM 切流样例：主上游 → K8s Ingress；备上游 → 现网 116 Docker vLLM
# 对照：patches_cursor/LiteLLM部署/config.yaml、作业17 主备路由
# 变更仍走：备份 → diff → 拷到 /opt/litellm → systemctl restart litellm → curl 验收

model_list:
  - model_name: qwen3.6-27b
    litellm_params:
      model: openai/qwen3.6-27b
      api_base: http://vllm-qwen.gpu-k8s.example.local/v1
      api_key: "sk-cluster-gpu"
      timeout: 300
      max_retries: 2
      extra_body:
        chat_template_kwargs:
          enable_thinking: false

  # 回滚锚点：现网 116 Docker vLLM
  - model_name: qwen3.6-27b-docker116
    litellm_params:
      model: openai/qwen3.6-27b
      api_base: http://116.62.10.183:8000/v1
      api_key: "123456"
      timeout: 300
      max_retries: 0

  - model_name: qwen-small
    litellm_params:
      model: openai/qwen-small
      api_base: http://vllm-small.gpu-k8s.example.local/v1
      api_key: "sk-cluster-gpu"
      timeout: 120
      max_retries: 1

litellm_settings:
  drop_params: true
  set_verbose: false
  request_timeout: 300
  num_retries: 2
  fallbacks:
    - qwen3.6-27b: [qwen3.6-27b-docker116]

router_settings:
  routing_strategy: simple-shuffle

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY

# GPU Operator Helm values 片段（纸面跟读；版本号以官网稳定版为准）
# helm upgrade --install gpu-operator nvidia/gpu-operator \
#   -n gpu-operator --create-namespace -f 07-gpu-operator-values.yaml

driver:
  enabled: true
toolkit:
  enabled: true
devicePlugin:
  enabled: true
dcgmExporter:
  enabled: true
gfd:
  enabled: true
migManager:
  enabled: false
# 若节点已由 OS 镜像固化驱动，可改 driver.enabled=false，只装 toolkit + plugin
operator:
  defaultRuntime: containerd