377 lines
21 KiB
Python
377 lines
21 KiB
Python
"""Deploy-Orchestrierung: Plan (wie infra/deploy.py), Ausführung über den Mittwald-Client.
|
||
|
||
Sicherheit: Ohne `DEPLOY_LIVE=true` (Env der Admin-Plattform) ist JEDE schreibende Operation gesperrt – es gibt nur Dry-Runs.
|
||
Außerdem blockiert der Core-Kompatibilitäts-Schalter (`core_supports_bundle`) Tenants mit Quelle „Admin“,
|
||
solange der Core das Bundle noch nicht laden kann.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from datetime import datetime, timezone
|
||
|
||
from sqlalchemy import select
|
||
|
||
from app.db import session
|
||
from app.models import Deployment, Secret, Setting, Tenant
|
||
from app.services import contract, crypto, guard, netcheck, onyva_login, store
|
||
from app.services.mittwald import DeployError, Mittwald, mask_env
|
||
|
||
PORT = "8276"
|
||
# Geteilte Secrets, die – falls nicht in der DB gesetzt – aus der Env der Admin-Plattform kommen (gleicher Key wie für Agent/Prompt-Test)
|
||
ENV_FALLBACK_SECRETS = ("CORTECS_API_KEY",)
|
||
|
||
|
||
def _env_secret(key: str) -> str:
|
||
return os.getenv(key, "").strip() if key in ENV_FALLBACK_SECRETS else ""
|
||
|
||
|
||
def live_enabled() -> bool:
|
||
"""DEPLOY_LIVE=true UND aktiver Login (guard.auth_ok) – ohne Login bleibt alles im Dry-Run."""
|
||
return os.getenv("DEPLOY_LIVE", "false").lower() == "true" and guard.auth_ok()
|
||
|
||
|
||
def token_present() -> bool:
|
||
return bool(os.getenv("MITTWALD_API_TOKEN", "").strip())
|
||
|
||
|
||
def setting(db, key: str, default: str = "") -> str:
|
||
s = db.get(Setting, key)
|
||
return s.value if s and s.value != "" else default
|
||
|
||
|
||
def host_for(db, t: Tenant) -> str:
|
||
if t.custom_domain and t.domain_status == "verified":
|
||
return t.custom_domain
|
||
return f"{t.subdomain or t.id}.{setting(db, 'base_domain', 'example.de')}"
|
||
|
||
|
||
def effective_tag(db, t: Tenant | None = None, override: str | None = None) -> str:
|
||
"""Welcher Core-Tag gilt? Override (Rollout) > Pin des Tenants > freigegebener Standard-Tag."""
|
||
return override or (t.core_pin if t is not None and t.core_pin else None) or setting(db, "core_image_tag", "latest")
|
||
|
||
|
||
def image_ref(db, t: Tenant | None = None, tag: str | None = None) -> str:
|
||
registry = setting(db, "registry", "").rstrip("/")
|
||
name = setting(db, "image_name", "transkript-platform")
|
||
base = f"{registry}/{name}" if registry else name
|
||
return f"{base.lower()}:{effective_tag(db, t, tag)}"
|
||
|
||
|
||
def build_env(db, t: Tenant, reveal: bool) -> dict[str, str]:
|
||
"""Container-Env, abgeleitet aus dem Env-Schema des Cores (kind/scope) – neue Core-Variablen erscheinen ohne Admin-Änderung.
|
||
reveal=False → Secrets als Platzhalter."""
|
||
host = host_for(db, t)
|
||
env = {"TENANT": t.id, "DATA_DIR": "/data", "PORT": PORT,
|
||
"OIDC_REDIRECT_URI": f"https://{host}/auth/callback", "OIDC_LOGOUT_REDIRECT_URI": f"https://{host}/"}
|
||
spec = contract.env_spec()
|
||
for e in spec: # geteilte, nicht geheime Einstellungen
|
||
if e["kind"] == "setting" and e["scope"] == "shared":
|
||
v = setting(db, e.get("setting") or e["name"].lower())
|
||
if v:
|
||
env[e["name"]] = v
|
||
shared = db.scalars(select(Secret).where(Secret.tenant_id.is_(None))).all()
|
||
own = db.scalars(select(Secret).where(Secret.tenant_id == t.id)).all()
|
||
for s_ in [*shared, *own]: # Tenant-Werte überschreiben geteilte; auch Secrets ohne Schema-Eintrag werden durchgereicht
|
||
if s_.key == "ADMIN_BUNDLE_TOKEN" and t.source != "admin":
|
||
continue
|
||
if s_.value_enc:
|
||
env[s_.key] = crypto.decrypt(s_.value_enc) if reveal else "•••• gesetzt"
|
||
elif not reveal:
|
||
env[s_.key] = "⚠ fehlt (Wert noch nicht in der Admin-DB)"
|
||
for k in ENV_FALLBACK_SECRETS: # Admin-Env als Rückfall für geteilte Secrets ohne DB-Wert
|
||
v = _env_secret(k)
|
||
if v and (k not in env or not env[k] or env[k].startswith("⚠")):
|
||
env[k] = v if reveal else "•••• gesetzt (aus Admin-Env)"
|
||
if t.source == "admin":
|
||
env["TENANT_SOURCE"] = "admin"
|
||
env["ADMIN_URL"] = setting(db, "admin_public_url", "https://<admin-host>")
|
||
return env
|
||
|
||
|
||
def missing_secrets(db, t: Tenant) -> list[str]:
|
||
"""Pflicht-Secrets laut Schema, die weder beim Tenant noch geteilt einen Wert haben."""
|
||
cfg, _ = store.content(db, t, "current")
|
||
have = {s.key for s in db.scalars(select(Secret).where((Secret.tenant_id == t.id) | Secret.tenant_id.is_(None))) if s.value_enc}
|
||
have |= {k for k in ENV_FALLBACK_SECRETS if _env_secret(k)}
|
||
need = set(contract.required_secrets(cfg))
|
||
if t.source == "admin":
|
||
need.add("ADMIN_BUNDLE_TOKEN")
|
||
return sorted(need - have)
|
||
|
||
|
||
def plan(db, t: Tenant) -> dict:
|
||
cfg, _ = store.content(db, t, "current")
|
||
return {
|
||
"service": f"app-{t.id}", "volume": f"vol-{t.id}",
|
||
"patch": {"description": cfg.get("name") or f"Onyscribe {t.id}", "image": image_ref(db, t),
|
||
"environment": build_env(db, t, reveal=False), "ports": [f"{PORT}:{PORT}/tcp"], "volumes": [f"vol-{t.id}:/data"]},
|
||
"ingress": {"host": host_for(db, t), "target": f"app-{t.id}:{PORT}"},
|
||
"missing_secrets": missing_secrets(db, t),
|
||
"oidc": {"mode": onyva_login.config_of(t)["mode"], "key": onyva_login.key_present(), "writes": onyva_login.writes_allowed(),
|
||
"last_sync": onyva_login.config_of(t)["last_sync"]},
|
||
"mode": {"live": live_enabled(), "token": token_present(), "tenant_allowed": guard.live_allowed(t.id), "allowed": guard.describe(), "project": bool(setting(db, "mittwald_project_id")),
|
||
"core_bundle": setting(db, "core_supports_bundle", "false") == "true"},
|
||
"core": {"tag": effective_tag(db, t), "pinned": bool(t.core_pin), "admin_core": contract.sha()[:10], "contract": contract.contract_version()},
|
||
}
|
||
|
||
|
||
# ── Ausführung ──────────────────────────────────────────────────────────────
|
||
|
||
def _append(dep_id: int, line: str) -> None:
|
||
with session() as db:
|
||
db.get(Deployment, dep_id).log += f"{datetime.now(timezone.utc).strftime('%H:%M:%S')} {line}\n"
|
||
|
||
|
||
def _finish(dep_id: int, status: str) -> None:
|
||
with session() as db:
|
||
d = db.get(Deployment, dep_id)
|
||
d.status, d.finished_at = status, datetime.now(timezone.utc)
|
||
|
||
|
||
def _client(live: bool, log, http=None) -> Mittwald:
|
||
return Mittwald(os.getenv("MITTWALD_API_TOKEN", "").strip(), os.getenv("MITTWALD_API_BASE", "https://api.mittwald.de/v2"), live, log, http)
|
||
|
||
|
||
def recheck_health(tenant_id: str, *, health=None, tries: int = 8, delay: float = 5.0) -> dict:
|
||
"""Health-Check nachholen (z. B. wenn das Zertifikat inzwischen da ist). Bei Erfolg wird ein hängender Deploy-Eintrag auf „success“ gesetzt."""
|
||
lines: list[str] = []
|
||
with session() as db:
|
||
t = db.get(Tenant, tenant_id)
|
||
host, tag, ver, source = host_for(db, t), effective_tag(db, t), t.current_version, t.source
|
||
deployed = (t.deployed_version, t.deployed_image)
|
||
try:
|
||
args = (host, deployed[1] or tag, ver if source == "admin" else None, lines.append)
|
||
health(*args) if health else verify_health(*args, tries=tries, delay=delay)
|
||
except DeployError as exc:
|
||
return {"ok": False, "log": lines + [f"✘ {exc}"]}
|
||
with session() as db:
|
||
d = db.scalar(select(Deployment).where(Deployment.tenant_id == tenant_id, Deployment.kind == "deploy").order_by(Deployment.id.desc()))
|
||
if d is not None and d.status in ("warning", "failed") and not d.simulated:
|
||
d.status = "success"
|
||
d.log += f"{datetime.now(timezone.utc).strftime('%H:%M:%S')} Health nachträglich ✔ – Deploy als erfolgreich markiert\n"
|
||
return {"ok": True, "log": lines}
|
||
|
||
|
||
def _hex_tag(tag: str) -> bool:
|
||
return bool(tag) and all(c in "0123456789abcdef" for c in tag.lower()) and len(tag) >= 7
|
||
|
||
|
||
class HealthTimeout(DeployError):
|
||
"""Health-Check nicht grün geworden. answered=False: es kam nie eine gültige Antwort (Zertifikat/DNS/Container noch nicht bereit)."""
|
||
|
||
def __init__(self, msg: str, answered: bool, dns: bool = False):
|
||
super().__init__(msg)
|
||
self.answered = answered
|
||
self.dns = dns # Hostname war vom Admin-Server aus nicht auflösbar (Resolver-Cache/DNS), nicht Fehler des Containers
|
||
|
||
|
||
def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, log, *, get=None, tries: int = 18, delay: float = 5.0,
|
||
sleep=time.sleep) -> None:
|
||
"""Prüft nach dem Deploy, ob wirklich läuft, was wir ausgerollt haben (Vertrag: /health.core / .bundle).
|
||
Wirft DeployError, wenn der Container nach `tries` Versuchen nicht gesund ist oder die falsche Version meldet."""
|
||
if get is None:
|
||
get = netcheck.get_json
|
||
last, answered, dns = "keine Antwort", False, False
|
||
for i in range(tries):
|
||
try:
|
||
status, body = get(f"https://{host}/health")
|
||
answered = answered or (status == 200 and isinstance(body, dict) and "status" in body) # echte Antwort der App (nicht Proxy/Zertifikatsfehler)
|
||
problems = []
|
||
if status != 200 or body.get("status") != "ok":
|
||
problems.append(f"HTTP {status}")
|
||
core = body.get("core")
|
||
if expect_sha and _hex_tag(expect_sha):
|
||
if core is None:
|
||
log("⚠ /health meldet keine Core-Version (Image vor dem Vertrag) – Version nicht prüfbar")
|
||
elif not contract.same_sha(core.get("version", ""), expect_sha):
|
||
problems.append(f"Core {str(core.get('version'))[:10]} statt {expect_sha[:10]}")
|
||
if expect_bundle is not None and body.get("bundle"):
|
||
b = body["bundle"]
|
||
if b.get("source") != "admin" or b.get("version") != expect_bundle or b.get("stale"):
|
||
problems.append(f"Bundle {b.get('source')} v{b.get('version')}{' (veraltet)' if b.get('stale') else ''} statt admin v{expect_bundle}"
|
||
+ (f" – {b.get('error')}" if b.get("error") else ""))
|
||
if not problems:
|
||
if body.get("_dns_fallback"):
|
||
log("(Hinweis: Hostname vom Admin-Server nicht auflösbar – Prüfung über DNS-over-HTTPS direkt zur IP)")
|
||
log(f"Health-Check https://{host}/health ✔" + (f" · Core {core['version'][:10]}" if core else "")
|
||
+ (f" · Bundle v{body['bundle']['version']}" if body.get("bundle", {}).get("source") == "admin" else ""))
|
||
return
|
||
last = "; ".join(problems)
|
||
except Exception as exc: # noqa: BLE001
|
||
last = f"{type(exc).__name__}: {exc}"
|
||
dns = netcheck.is_dns_failure_text(last)
|
||
if i < tries - 1:
|
||
sleep(delay)
|
||
raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered, dns and not answered)
|
||
|
||
|
||
def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None, strict: bool = False) -> bool:
|
||
"""Führt einen Deployment-Datensatz aus. kind: dry-run | deploy | offboard-dry | offboard.
|
||
core_tag überschreibt den Core-Tag (Rollout/Rollback); `health(host, expect_sha, expect_bundle, log)` ist in Tests ersetzbar."""
|
||
log = lambda m: _append(dep_id, m) # noqa: E731
|
||
dry = kind in ("dry-run", "offboard-dry")
|
||
live = live_enabled() and not dry
|
||
try:
|
||
with session() as db:
|
||
t = db.get(Tenant, tenant_id)
|
||
ver = t.current_version
|
||
project, stack = setting(db, "mittwald_project_id"), setting(db, "mittwald_stack_id")
|
||
host = host_for(db, t) if setting(db, "base_domain") else None
|
||
source, bundle_ok = t.source, setting(db, "core_supports_bundle", "false") == "true"
|
||
image, tag = image_ref(db, t, core_tag), effective_tag(db, t, core_tag)
|
||
pinned = bool(t.core_pin) and not core_tag
|
||
log(f"▶ {kind} „{tenant_id}“ · Version v{ver} · {image}{' (gepinnt)' if pinned else ''} · Modus: {'LIVE' if live else 'Dry-Run (keine Schreibzugriffe)'}")
|
||
if not contract.supported():
|
||
raise DeployError(f"Core-Vertrag {contract.contract_version()} wird von dieser Admin-Version nicht unterstützt – Admin aktualisieren.")
|
||
if source == "admin" and not bundle_ok and not dry:
|
||
raise DeployError("Quelle „Admin“ ist noch gesperrt: Der Core kann das Bundle noch nicht laden (Einstellung core_supports_bundle).")
|
||
if not dry and not live_enabled():
|
||
raise DeployError("Echte Deployments sind gesperrt (DEPLOY_LIVE != true).")
|
||
if not dry and not guard.live_allowed(tenant_id):
|
||
raise DeployError(guard.denial(tenant_id))
|
||
if live and source == "admin":
|
||
with session() as db:
|
||
url = setting(db, "admin_public_url")
|
||
if not url.startswith("https://") or "<" in url:
|
||
raise DeployError("Einstellung „Öffentliche URL der Admin-Plattform“ (https://…) fehlt – der Container könnte sein Bundle nicht holen.")
|
||
|
||
if not kind.startswith("offboard"):
|
||
# 1) OIDC-Client abgleichen (legt bei Bedarf an) – VOR dem Service, weil dessen Env die Client-Daten braucht
|
||
rep = onyva_login.sync_tenant(tenant_id, apply=live, log=log, client=oidc_client, actor="deploy")
|
||
if live and not rep["ok"]:
|
||
raise DeployError(f"OIDC-Abgleich fehlgeschlagen: {rep['error']}")
|
||
with session() as db:
|
||
t = db.get(Tenant, tenant_id)
|
||
if live:
|
||
lacking = missing_secrets(db, t)
|
||
if lacking:
|
||
raise DeployError("Pflicht-Secrets ohne Wert: " + ", ".join(lacking) + " (Tenant → Secrets bzw. Domain & Login, geteilte unter Einstellungen).")
|
||
definition = {"description": plan(db, t)["patch"]["description"], "image": image, "ports": [f"{PORT}:{PORT}/tcp"],
|
||
"volumes": [f"vol-{t.id}:/data"], "environment": build_env(db, t, reveal=live)}
|
||
mw = _client(live, log, http)
|
||
sid = mw.resolve_stack_id(project, stack)
|
||
if kind.startswith("offboard"):
|
||
mw.offboard_service(stack_id=sid, service=f"app-{tenant_id}", volume=f"vol-{tenant_id}", project_id=project, hostname=host)
|
||
log("Hinweis: Der OIDC-Client bleibt im Login bestehen (Tenant → Übersicht → Gefahrenbereich: „OIDC-Client im Login löschen“).")
|
||
else:
|
||
mw.deploy_service(stack_id=sid, service=f"app-{tenant_id}", volume=f"vol-{tenant_id}", definition=definition,
|
||
container_port=PORT, project_id=project, hostname=host)
|
||
warning = False
|
||
if not dry: # Zustand sofort festhalten: geschrieben ist geschrieben, auch wenn der Health-Check noch wartet
|
||
with session() as db:
|
||
t = db.get(Tenant, tenant_id)
|
||
if kind == "deploy":
|
||
t.deployed_version, t.deployed_image, t.deployed_at = ver, tag, datetime.now(timezone.utc)
|
||
store.audit(db, "system", f"{kind}.live", tenant_id, f"v{ver} :{tag}")
|
||
if live and not kind.startswith("offboard") and host:
|
||
fresh = mw.ingress_created
|
||
# Neuer Ingress: Zertifikat/DNS bei Mittwald brauchen oft mehrere Minuten → geduldig warten (bis ~15 min)
|
||
kw = {"tries": 90, "delay": 10.0} if fresh else {}
|
||
if fresh:
|
||
log("Neuer Ingress angelegt – Zertifikat/DNS bei Mittwald können einige Minuten dauern, Health-Check wartet geduldig …")
|
||
try:
|
||
args = (host, tag, ver if source == "admin" else None, log)
|
||
health(*args) if health else verify_health(*args, **kw)
|
||
except HealthTimeout as exc:
|
||
# Warnung statt Fehler: neuer Ingress ohne Antwort (Zertifikat/DNS) – oder (außerhalb strikter Rollouts) ein Namensauflösungsproblem des Admin-Servers
|
||
if not exc.answered and (fresh or (exc.dns and not strict)):
|
||
warning = True
|
||
log(f"⚠ {exc}")
|
||
if exc.dns:
|
||
log("⚠ Der Admin-Server kann den Hostnamen nicht auflösen (Resolver-Cache/DNS) – der Deploy selbst ist geschrieben. "
|
||
"Prüfe im Browser, ob die Adresse läuft, und später „Health erneut prüfen“.")
|
||
else:
|
||
log("⚠ Container und Ingress sind angelegt, aber die Adresse antwortet noch nicht (Zertifikat/DNS?). "
|
||
"Später „Health erneut prüfen“ (Deploy-Tab) oder „Läuft?“ (Übersicht).")
|
||
else:
|
||
raise
|
||
log("✔ Fertig" if live and not warning else ("Fertig mit Warnung" if warning else "Dry-Run beendet – es wurde nichts geschrieben."))
|
||
return "warning" if warning else True
|
||
except DeployError as exc:
|
||
log(f"✘ {exc}")
|
||
return False
|
||
except Exception as exc: # noqa: BLE001
|
||
log(f"✘ Unerwarteter Fehler: {type(exc).__name__}: {exc}")
|
||
return False
|
||
|
||
|
||
def start(tenant_id: str, kind: str, actor: str, batch_id: str = "", http=None, wait: bool = False, oidc_client=None,
|
||
core_tag: str | None = None, health=None) -> int:
|
||
with session() as db:
|
||
t = db.get(Tenant, tenant_id)
|
||
d = Deployment(tenant_id=tenant_id, kind=kind, version=t.current_version, image_tag=effective_tag(db, t, core_tag),
|
||
actor=actor, batch_id=batch_id, simulated=not (live_enabled() and kind in ("deploy", "offboard")))
|
||
db.add(d)
|
||
db.flush()
|
||
dep_id = d.id
|
||
|
||
def work():
|
||
ok = execute(dep_id, tenant_id, kind, http, oidc_client, core_tag, health)
|
||
_finish(dep_id, "warning" if ok == "warning" else ("success" if ok else "failed"))
|
||
|
||
th = threading.Thread(target=work, daemon=True)
|
||
th.start()
|
||
if wait:
|
||
th.join()
|
||
return dep_id
|
||
|
||
|
||
def run_sequence(tenant_ids: list[str], kind: str, actor: str, *, batch: str, core_tag: str | None = None, http=None, oidc_client=None,
|
||
health=None, canary: list[str] | None = None, wait: bool = False, on_done=None) -> list[int]:
|
||
"""Tenants nacheinander (zuerst die Canary-Tenants), Abbruch beim ersten Fehler – der Rest wird „cancelled“.
|
||
on_done(ok, results) wird nach dem Lauf aufgerufen (z. B. um ein Release abzuschließen)."""
|
||
if kind in ("deploy", "offboard") and live_enabled(): # Live: nur freigegebene Tenants (siehe guard.py)
|
||
skipped = [t for t in tenant_ids if not guard.live_allowed(t)]
|
||
tenant_ids = [t for t in tenant_ids if guard.live_allowed(t)]
|
||
if skipped:
|
||
with session() as db:
|
||
store.audit(db, actor, "deploy.skipped_not_allowed", None, ", ".join(skipped))
|
||
canary = [c for c in (canary or []) if c in tenant_ids]
|
||
order = canary + [t for t in tenant_ids if t not in canary]
|
||
with session() as db:
|
||
rows = []
|
||
for tid in order:
|
||
t = db.get(Tenant, tid)
|
||
d = Deployment(tenant_id=tid, kind=kind, version=t.current_version, image_tag=effective_tag(db, t, core_tag), actor=actor,
|
||
batch_id=batch, status="queued", simulated=not (live_enabled() and kind == "deploy"))
|
||
db.add(d)
|
||
rows.append(d)
|
||
db.flush()
|
||
dep_ids = [d.id for d in rows]
|
||
|
||
def seq():
|
||
failed, results = False, {}
|
||
for tid, did in zip(order, dep_ids):
|
||
if failed:
|
||
_append(did, "übersprungen – vorheriger Tenant fehlgeschlagen (Rollout abgebrochen)")
|
||
_finish(did, "cancelled")
|
||
results[tid] = "cancelled"
|
||
continue
|
||
with session() as db:
|
||
db.get(Deployment, did).status = "running"
|
||
ok = execute(did, tid, kind, http, oidc_client, core_tag, health, strict=bool(core_tag))
|
||
_finish(did, "warning" if ok == "warning" else ("success" if ok else "failed"))
|
||
results[tid] = "warning" if ok == "warning" else ("success" if ok else "failed")
|
||
failed = not ok
|
||
if tid in canary and ok and tid == canary[-1]:
|
||
_append(did, "Canary erfolgreich – restliche Tenants folgen")
|
||
if on_done:
|
||
on_done(not failed, results)
|
||
|
||
th = threading.Thread(target=seq, daemon=True)
|
||
th.start()
|
||
if wait:
|
||
th.join()
|
||
return dep_ids
|
||
|
||
|
||
def start_all(kind: str, actor: str, http=None, wait: bool = False, oidc_client=None, core_tag: str | None = None, health=None) -> tuple[str, list[int]]:
|
||
"""Alle aktiven, nicht gepinnten Tenants nacheinander; beim ersten Fehler Abbruch."""
|
||
batch = uuid.uuid4().hex[:10]
|
||
with session() as db:
|
||
ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status == "active").order_by(Tenant.id)) if not (core_tag and t.core_pin)]
|
||
return batch, run_sequence(ids, kind, actor, batch=batch, core_tag=core_tag, http=http, oidc_client=oidc_client, health=health, wait=wait)
|