#!/bin/bash
set -u
backup=/usr/cluster/k8s-reset-backup-20261012
mkdir -p "$backup"
iptables-save > "$backup/iptables.rules"
ip6tables-save > "$backup/ip6tables.rules" || true
ipvsadm-save > "$backup/ipvs.rules" 2>/dev/null || true
ip route > "$backup/route.txt"
ip -br addr > "$backup/addr.txt"
docker ps --format '{{.Names}}' | sort > "$backup/docker-before.txt"
echo "backup $backup"

check_health() {
  local label=$1
  local ok=1
  local u code
  for u in \
    http://127.0.0.1:3000/api/status \
    http://127.0.0.1:8000/health \
    http://127.0.0.1:8088/ \
    http://127.0.0.1:3100/api/public/health \
    http://127.0.0.1:3001/api/health \
    http://127.0.0.1:9090/-/ready \
    http://127.0.0.1:8090/ \
    http://127.0.0.1:80/
  do
    code=$(curl -s -o /dev/null -m 8 -w '%{http_code}' "$u" || echo err)
    echo "HEALTH $label $code $u"
    case "$code" in
      200|301|302|307|308) ;;
      *) ok=0 ;;
    esac
  done
  if ! docker ps --format '{{.Names}}' | grep -qx 'vllm-qwen3.6-27b'; then
    echo "HEALTH $label vllm container missing"
    ok=0
  fi
  if [ "$ok" -ne 1 ]; then
    echo "HEALTH_FAIL $label"
    return 1
  fi
  echo "HEALTH_OK $label"
}

check_health before || exit 1

echo "=== stop kubelet ==="
systemctl stop kubelet
systemctl disable kubelet

echo "=== remove k8s containers only ==="
mapfile -t kids < <(docker ps -aq --filter name='k8s_')
if [ "${#kids[@]}" -gt 0 ]; then
  docker rm -f "${kids[@]}"
fi
docker ps --format '{{.Names}}' | grep '^k8s_' && echo "k8s containers remain" && exit 1
echo "k8s containers removed: ${#kids[@]}"

check_health after-rm || {
  echo "restore not needed yet; docker health failed after removing k8s containers"
  exit 1
}

echo "=== kubeadm reset ==="
timeout 90 kubeadm reset -f \
  --cri-socket unix:///var/run/docker.sock \
  --skip-phases=update-cluster-status,remove-etcd-member
echo "reset_exit=$?"

docker ps --format '{{.Names}}' | sort > "$backup/docker-after-reset.txt"
echo "=== container diff ==="
comm -23 "$backup/docker-before.txt" "$backup/docker-after-reset.txt" | grep -v '^k8s_' > "$backup/unexpected-missing.txt" || true
if [ -s "$backup/unexpected-missing.txt" ]; then
  echo "UNEXPECTED_MISSING"
  cat "$backup/unexpected-missing.txt"
  exit 1
fi

check_health after-reset || {
  echo "restoring iptables"
  iptables-restore < "$backup/iptables.rules" || true
  exit 1
}

echo "=== drop leftover calico links and routes ==="
ip route del blackhole 10.243.212.128/26 2>/dev/null || true
ipvsadm --clear 2>/dev/null || true
ip link del kube-ipvs0 2>/dev/null || true
ip link set tunl0 down 2>/dev/null || true
ip link del tunl0 2>/dev/null || true
while read -r nic; do
  [ -n "$nic" ] || continue
  ip link del "$nic" 2>/dev/null || true
done < <(ip -o link show | awk -F': ' '/cali|tunl|flannel|cni0|kube-ipvs/{print $2}' | cut -d@ -f1)

python3 - << 'PY'
import subprocess
raw = subprocess.check_output(["iptables-save"], text=True)
out = []
for line in raw.splitlines():
    if "cali-" in line or "CALI-" in line or "KUBE-" in line or "KUBE:" in line:
        continue
    out.append(line)
path = "/tmp/iptables.nokube"
open(path, "w").write("\n".join(out) + "\n")
print("filtered lines", len(raw.splitlines()), "->", len(out))
PY
if ! iptables-restore < /tmp/iptables.nokube; then
  echo "iptables-restore failed, restoring backup"
  iptables-restore < "$backup/iptables.rules" || true
  exit 1
fi

check_health after-net || {
  echo "restoring iptables after net cleanup"
  iptables-restore < "$backup/iptables.rules" || true
  exit 1
}

echo "=== leftover ifaces ==="
ip -br link
echo "=== leftover routes ==="
ip route
echo "CLEAN_OK"