#!/usr/bin/env python3 # -*- coding: utf-8 -*- """作业17 · 根据 Deployment YAML 生成「发布检查清单」PR 模板 用法: python hw17_01_发布检查清单PR模板.py python hw17_01_发布检查清单PR模板.py \ --deployment patches_cursor/作业17_K8s与模型路由/vllm-qwen-deployment.yaml \ --routing patches_cursor/作业17_K8s与模型路由/litellm-routing-primary-backup.yaml \ --author 完颜振江 --ticket OPS-1234 输出: data/homework17/pr_release_checklist_.md """ from __future__ import annotations import argparse import os import re from datetime import datetime from pathlib import Path from typing import Any def out_dir() -> Path: p = Path(os.environ.get("HW17_OUT", Path(__file__).resolve().parents[2] / "data" / "homework17")) p.mkdir(parents=True, exist_ok=True) return p def parse_deployment_yaml(text: str) -> dict[str, Any]: """轻量解析 Deployment 关键字段(不依赖 PyYAML)。""" # 只解析第一个 Deployment 块 dep_block = text m = re.search(r"kind:\s*Deployment\s*\n([\s\S]*?)(?:\n---|\Z)", text) if m: dep_block = "kind: Deployment\n" + m.group(1) info: dict[str, Any] = { "name": _match(r"^\s*name:\s*(\S+)", dep_block, multiline=True), "namespace": _match(r"^\s*namespace:\s*(\S+)", dep_block, multiline=True), "replicas": _match(r"^\s*replicas:\s*(\d+)", dep_block) or "1", "image": None, "gpu_limit": None, "max_num_seqs": None, "gpu_mem_util": None, "release_id": _match(r"config/release-id:\s*\"([^\"]+)\"", dep_block), "owner": _match(r"config/owner:\s*\"([^\"]+)\"", dep_block), "labels": {}, } m_img = re.search(r"containers:\s*\n(?:.*\n)*?\s+-\s+name:\s*\S+\s*\n\s+image:\s*(\S+)", dep_block) if m_img: info["image"] = m_img.group(1) info["gpu_limit"] = _match(r"nvidia\.com/gpu:\s*\"?(\d+)\"?", dep_block) info["max_num_seqs"] = _match(r"--max-num-seqs\s*\n\s*-\s*\"?(\d+)\"?", dep_block) info["gpu_mem_util"] = _match(r"--gpu-memory-utilization\s*\n\s*-\s*\"?([\d.]+)\"?", dep_block) for key in ("app", "model", "release-track"): v = _match(rf"^\s+{key}:\s*(\S+)", dep_block, flags=re.MULTILINE) if v: info["labels"][key] = v.strip('"') return info def parse_routing_yaml(text: str) -> dict[str, Any]: models = re.findall(r"model_name:\s*(\S+)", text) fallbacks = re.findall(r"-\s*(\S+):\s*\[([^\]]+)\]", text) bases = re.findall(r"api_base:\s*(\S+)", text) return { "model_names": models, "fallbacks": [{"from": a, "to": b.strip()} for a, b in fallbacks], "api_bases": bases, } def _match(pattern: str, text: str, multiline: bool = False, flags: int = 0) -> str | None: f = flags | (re.MULTILINE if multiline else 0) m = re.search(pattern, text, f) return m.group(1) if m else None def build_pr_body( dep: dict[str, Any], routing: dict[str, Any] | None, meta: dict[str, str], ) -> str: dep_name = dep.get("name") or "unknown-deployment" ns = dep.get("namespace") or "default" image = dep.get("image") or "TBD" gpu = dep.get("gpu_limit") or "?" release_id = dep.get("release_id") or meta.get("release_id", "TBD") routing_section = "" if routing: routing_section = f""" ## 模型路由变更(LiteLLM / 网关) | 项 | 值 | |----|-----| | 虚拟模型 | `{", ".join(routing.get("model_names") or [])}` | | upstream | `{", ".join(routing.get("api_bases") or [])}` | | fallbacks | `{routing.get("fallbacks")}` | - [ ] 已在 **测试** LiteLLM(`:4010` 或 staging)验证 primary / backup - [ ] 生产变更已 **备份** 上一版 `config.yaml`(路径 + 时间戳) - [ ] 变更记录写入审计表(操作人 / ticket / diff 摘要) """ return f"""## Summary 发布 **{dep_name}**(namespace `{ns}`)及关联模型路由配置。 Release ID: `{release_id}` · Ticket: `{meta.get("ticket", "TBD")}` ## 变更范围 | 项 | 当前值 | |----|--------| | Deployment | `{dep_name}` | | Namespace | `{ns}` | | Image | `{image}` | | Replicas | `{dep.get("replicas")}` | | GPU limit | `{gpu}` × `nvidia.com/gpu` | | max-num-seqs | `{dep.get("max_num_seqs") or "见 args"}` | | gpu-memory-utilization | `{dep.get("gpu_mem_util") or "见 args"}` | | Owner | `{dep.get("owner") or meta.get("author", "TBD")}` | {routing_section} ## 发布前检查清单(必勾) ### GPU / 集群 - [ ] 已确认 GPU 型号与驱动:`nvidia-smi` / `kubectl describe node` 标签 - [ ] **NVIDIA GPU Operator** 已安装:`kubectl get pods -n gpu-operator` 全 Running - [ ] Device Plugin 正常:`kubectl get nodes -o json | jq '.items[].status.allocatable["nvidia.com/gpu"]'` - [ ] 目标节点有足够 **allocatable GPU**(≥ `{gpu}`) ### 配置与镜像 - [ ] Deployment YAML / ConfigMap diff 已 review(无意外改 replicas/资源) - [ ] 镜像 tag 已 pin(非裸 `latest`,或已记录 digest) - [ ] 模型路由 **主备** 与 fallbacks 符合 Runbook(作业17 / 作业13) - [ ] Secrets / api_key 未提交到 Git 明文 ### 发布执行 - [ ] 低峰窗口 / 已通知业务 - [ ] `kubectl apply --dry-run=server -f ...` 通过 - [ ] 滚动发布或 `kubectl rollout restart deployment/{dep_name} -n {ns}` - [ ] `kubectl rollout status deployment/{dep_name} -n {ns}` 成功 ### 发布后验收 - [ ] Pod Running 且 **无 NVIDIA 相关 Event**(见下方命令) - [ ] `curl` /chat/completions 探活(primary 模型) - [ ] 故意停 primary 或模拟 5xx,验证 **backup / fallback**(仅 staging 或预案演练) - [ ] Prometheus:`vllm:num_requests_waiting` 无异常飙升 - [ ] Langfuse 抽样 Trace 正常 ### 回滚预案 - [ ] 上一版 Deployment manifest / LiteLLM config 已 tag 或分支备份 - [ ] 回滚命令已写在 PR:`kubectl rollout undo deployment/{dep_name} -n {ns}` - [ ] 路由回滚:恢复 config.yaml + `docker compose restart litellm`(207 现网路径) ## kubectl · NVIDIA 失败速查(发布日执行) ```bash # Pod 事件(重点看 FailedScheduling / FailedMount / insufficient nvidia.com/gpu) kubectl describe pod -n {ns} -l app={dep.get("labels", {}).get("app", "vllm-qwen")} # 集群事件过滤 NVIDIA / GPU kubectl get events -n {ns} --sort-by='.lastTimestamp' | grep -iE 'nvidia|gpu|cuda|device-plugin|insufficient' # Operator 健康 kubectl get pods -n gpu-operator kubectl logs -n gpu-operator -l app=nvidia-device-plugin-daemonset --tail=50 ``` ## Test plan - [ ] Staging 部署与探活 - [ ] 主备切换演练(或 dry-run 文档) - [ ] 回滚演练(undo + 配置 restore)在隔离环境至少走一次 ## 审计记录(合并后填写) | 字段 | 值 | |------|-----| | 操作人 | {meta.get("author", "")} | | 时间 | {meta.get("time", "")} | | Ticket | {meta.get("ticket", "")} | | Release ID | {release_id} | | 变更摘要 | | | 回滚是否测试 | 是 / 否 | --- *本 PR 描述由 `hw17_01_发布检查清单PR模板.py` 根据 Deployment 自动生成,发布前请人工补全勾选。* """ def main() -> None: parser = argparse.ArgumentParser(description="生成发布检查清单 PR 模板") repo = Path(__file__).resolve().parents[2] parser.add_argument( "--deployment", default=str(repo / "patches_cursor" / "作业17_K8s与模型路由" / "vllm-qwen-deployment.yaml"), ) parser.add_argument( "--routing", default=str(repo / "patches_cursor" / "作业17_K8s与模型路由" / "litellm-routing-primary-backup.yaml"), ) parser.add_argument("--author", default=os.environ.get("GIT_AUTHOR", "platform-oncall")) parser.add_argument("--ticket", default="OPS-0000") parser.add_argument("--release-id", default=datetime.now().strftime("%Y%m%d-%H%M")) args = parser.parse_args() dep_path = Path(args.deployment) dep_text = dep_path.read_text(encoding="utf-8") dep = parse_deployment_yaml(dep_text) routing = None routing_path = Path(args.routing) if routing_path.exists(): routing = parse_routing_yaml(routing_path.read_text(encoding="utf-8")) meta = { "author": args.author, "ticket": args.ticket, "release_id": args.release_id, "time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } body = build_pr_body(dep, routing, meta) od = out_dir() out = od / f"pr_release_checklist_{datetime.now().strftime('%Y%m%d_%H%M%S')}.md" title = f"[Release] {dep.get('name', 'deployment')} · {args.release_id}" full = f"# {title}\n\n{body}" out.write_text(full, encoding="utf-8") print(f"Wrote {out}") print(f"\nPR Title: {title}\n") print(body[:1200] + "\n...\n") if __name__ == "__main__": main()