"""Deploy-Orchestrierung: Plan (wie infra/deploy.py), Ausführung über den Mittwald-Client. Sicherheit: Ohne `DEPLOY_LIVE=true` (Env der Admin-Plattform) ist JEDE schreibende Operation gesperrt – es gibt nur Dry-Runs. Außerdem blockiert der Core-Kompatibilitäts-Schalter (`core_supports_bundle`) Tenants mit Quelle „Admin“, solange der Core das Bundle noch nicht laden kann. """ from __future__ import annotations import os import threading import time import uuid from datetime import datetime, timezone from sqlalchemy import select from app.db import session from app.models import Deployment, Secret, Setting, Tenant from app.services import contract, crypto, guard, onyva_login, store from app.services.mittwald import DeployError, Mittwald, mask_env PORT = "8276" # Geteilte Secrets, die – falls nicht in der DB gesetzt – aus der Env der Admin-Plattform kommen (gleicher Key wie für Agent/Prompt-Test) ENV_FALLBACK_SECRETS = ("CORTECS_API_KEY",) def _env_secret(key: str) -> str: return os.getenv(key, "").strip() if key in ENV_FALLBACK_SECRETS else "" def live_enabled() -> bool: """DEPLOY_LIVE=true UND aktiver Login (guard.auth_ok) – ohne Login bleibt alles im Dry-Run.""" return os.getenv("DEPLOY_LIVE", "false").lower() == "true" and guard.auth_ok() def token_present() -> bool: return bool(os.getenv("MITTWALD_API_TOKEN", "").strip()) def setting(db, key: str, default: str = "") -> str: s = db.get(Setting, key) return s.value if s and s.value != "" else default def host_for(db, t: Tenant) -> str: if t.custom_domain and t.domain_status == "verified": return t.custom_domain return f"{t.subdomain or t.id}.{setting(db, 'base_domain', 'example.de')}" def effective_tag(db, t: Tenant | None = None, override: str | None = None) -> str: """Welcher Core-Tag gilt? Override (Rollout) > Pin des Tenants > freigegebener Standard-Tag.""" return override or (t.core_pin if t is not None and t.core_pin else None) or setting(db, "core_image_tag", "latest") def image_ref(db, t: Tenant | None = None, tag: str | None = None) -> str: registry = setting(db, "registry", "").rstrip("/") name = setting(db, "image_name", "transkript-platform") base = f"{registry}/{name}" if registry else name return f"{base.lower()}:{effective_tag(db, t, tag)}" def build_env(db, t: Tenant, reveal: bool) -> dict[str, str]: """Container-Env, abgeleitet aus dem Env-Schema des Cores (kind/scope) – neue Core-Variablen erscheinen ohne Admin-Änderung. reveal=False → Secrets als Platzhalter.""" host = host_for(db, t) env = {"TENANT": t.id, "DATA_DIR": "/data", "PORT": PORT, "OIDC_REDIRECT_URI": f"https://{host}/auth/callback", "OIDC_LOGOUT_REDIRECT_URI": f"https://{host}/"} spec = contract.env_spec() for e in spec: # geteilte, nicht geheime Einstellungen if e["kind"] == "setting" and e["scope"] == "shared": v = setting(db, e.get("setting") or e["name"].lower()) if v: env[e["name"]] = v shared = db.scalars(select(Secret).where(Secret.tenant_id.is_(None))).all() own = db.scalars(select(Secret).where(Secret.tenant_id == t.id)).all() for s_ in [*shared, *own]: # Tenant-Werte überschreiben geteilte; auch Secrets ohne Schema-Eintrag werden durchgereicht if s_.key == "ADMIN_BUNDLE_TOKEN" and t.source != "admin": continue if s_.value_enc: env[s_.key] = crypto.decrypt(s_.value_enc) if reveal else "•••• gesetzt" elif not reveal: env[s_.key] = "⚠ fehlt (Wert noch nicht in der Admin-DB)" for k in ENV_FALLBACK_SECRETS: # Admin-Env als Rückfall für geteilte Secrets ohne DB-Wert v = _env_secret(k) if v and (k not in env or not env[k] or env[k].startswith("⚠")): env[k] = v if reveal else "•••• gesetzt (aus Admin-Env)" if t.source == "admin": env["TENANT_SOURCE"] = "admin" env["ADMIN_URL"] = setting(db, "admin_public_url", "https://") return env def missing_secrets(db, t: Tenant) -> list[str]: """Pflicht-Secrets laut Schema, die weder beim Tenant noch geteilt einen Wert haben.""" cfg, _ = store.content(db, t, "current") have = {s.key for s in db.scalars(select(Secret).where((Secret.tenant_id == t.id) | Secret.tenant_id.is_(None))) if s.value_enc} have |= {k for k in ENV_FALLBACK_SECRETS if _env_secret(k)} need = set(contract.required_secrets(cfg)) if t.source == "admin": need.add("ADMIN_BUNDLE_TOKEN") return sorted(need - have) def plan(db, t: Tenant) -> dict: cfg, _ = store.content(db, t, "current") return { "service": f"app-{t.id}", "volume": f"vol-{t.id}", "patch": {"description": cfg.get("name") or f"Onyscribe {t.id}", "image": image_ref(db, t), "environment": build_env(db, t, reveal=False), "ports": [f"{PORT}:{PORT}/tcp"], "volumes": [f"vol-{t.id}:/data"]}, "ingress": {"host": host_for(db, t), "target": f"app-{t.id}:{PORT}"}, "missing_secrets": missing_secrets(db, t), "oidc": {"mode": onyva_login.config_of(t)["mode"], "key": onyva_login.key_present(), "writes": onyva_login.writes_allowed(), "last_sync": onyva_login.config_of(t)["last_sync"]}, "mode": {"live": live_enabled(), "token": token_present(), "tenant_allowed": guard.live_allowed(t.id), "allowed": guard.describe(), "project": bool(setting(db, "mittwald_project_id")), "core_bundle": setting(db, "core_supports_bundle", "false") == "true"}, "core": {"tag": effective_tag(db, t), "pinned": bool(t.core_pin), "admin_core": contract.sha()[:10], "contract": contract.contract_version()}, } # ── Ausführung ────────────────────────────────────────────────────────────── def _append(dep_id: int, line: str) -> None: with session() as db: db.get(Deployment, dep_id).log += f"{datetime.now(timezone.utc).strftime('%H:%M:%S')} {line}\n" def _finish(dep_id: int, status: str) -> None: with session() as db: d = db.get(Deployment, dep_id) d.status, d.finished_at = status, datetime.now(timezone.utc) def _client(live: bool, log, http=None) -> Mittwald: return Mittwald(os.getenv("MITTWALD_API_TOKEN", "").strip(), os.getenv("MITTWALD_API_BASE", "https://api.mittwald.de/v2"), live, log, http) def recheck_health(tenant_id: str, *, health=None, tries: int = 8, delay: float = 5.0) -> dict: """Health-Check nachholen (z. B. wenn das Zertifikat inzwischen da ist). Bei Erfolg wird ein hängender Deploy-Eintrag auf „success“ gesetzt.""" lines: list[str] = [] with session() as db: t = db.get(Tenant, tenant_id) host, tag, ver, source = host_for(db, t), effective_tag(db, t), t.current_version, t.source deployed = (t.deployed_version, t.deployed_image) try: args = (host, deployed[1] or tag, ver if source == "admin" else None, lines.append) health(*args) if health else verify_health(*args, tries=tries, delay=delay) except DeployError as exc: return {"ok": False, "log": lines + [f"✘ {exc}"]} with session() as db: d = db.scalar(select(Deployment).where(Deployment.tenant_id == tenant_id, Deployment.kind == "deploy").order_by(Deployment.id.desc())) if d is not None and d.status in ("warning", "failed") and not d.simulated: d.status = "success" d.log += f"{datetime.now(timezone.utc).strftime('%H:%M:%S')} Health nachträglich ✔ – Deploy als erfolgreich markiert\n" return {"ok": True, "log": lines} def _hex_tag(tag: str) -> bool: return bool(tag) and all(c in "0123456789abcdef" for c in tag.lower()) and len(tag) >= 7 class HealthTimeout(DeployError): """Health-Check nicht grün geworden. answered=False: es kam nie eine gültige Antwort (Zertifikat/DNS/Container noch nicht bereit).""" def __init__(self, msg: str, answered: bool): super().__init__(msg) self.answered = answered def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, log, *, get=None, tries: int = 18, delay: float = 5.0, sleep=time.sleep) -> None: """Prüft nach dem Deploy, ob wirklich läuft, was wir ausgerollt haben (Vertrag: /health.core / .bundle). Wirft DeployError, wenn der Container nach `tries` Versuchen nicht gesund ist oder die falsche Version meldet.""" if get is None: import httpx def get(url): r = httpx.get(url, timeout=8, follow_redirects=True) return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {}) last, answered = "keine Antwort", False for i in range(tries): try: status, body = get(f"https://{host}/health") answered = answered or (status == 200 and isinstance(body, dict) and "status" in body) # echte Antwort der App (nicht Proxy/Zertifikatsfehler) problems = [] if status != 200 or body.get("status") != "ok": problems.append(f"HTTP {status}") core = body.get("core") if expect_sha and _hex_tag(expect_sha): if core is None: log("⚠ /health meldet keine Core-Version (Image vor dem Vertrag) – Version nicht prüfbar") elif not contract.same_sha(core.get("version", ""), expect_sha): problems.append(f"Core {str(core.get('version'))[:10]} statt {expect_sha[:10]}") if expect_bundle is not None and body.get("bundle"): b = body["bundle"] if b.get("source") != "admin" or b.get("version") != expect_bundle or b.get("stale"): problems.append(f"Bundle {b.get('source')} v{b.get('version')}{' (veraltet)' if b.get('stale') else ''} statt admin v{expect_bundle}" + (f" – {b.get('error')}" if b.get("error") else "")) if not problems: log(f"Health-Check https://{host}/health ✔" + (f" · Core {core['version'][:10]}" if core else "") + (f" · Bundle v{body['bundle']['version']}" if body.get("bundle", {}).get("source") == "admin" else "")) return last = "; ".join(problems) except Exception as exc: # noqa: BLE001 last = f"{type(exc).__name__}: {exc}" if i < tries - 1: sleep(delay) raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered) def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None) -> bool: """Führt einen Deployment-Datensatz aus. kind: dry-run | deploy | offboard-dry | offboard. core_tag überschreibt den Core-Tag (Rollout/Rollback); `health(host, expect_sha, expect_bundle, log)` ist in Tests ersetzbar.""" log = lambda m: _append(dep_id, m) # noqa: E731 dry = kind in ("dry-run", "offboard-dry") live = live_enabled() and not dry try: with session() as db: t = db.get(Tenant, tenant_id) ver = t.current_version project, stack = setting(db, "mittwald_project_id"), setting(db, "mittwald_stack_id") host = host_for(db, t) if setting(db, "base_domain") else None source, bundle_ok = t.source, setting(db, "core_supports_bundle", "false") == "true" image, tag = image_ref(db, t, core_tag), effective_tag(db, t, core_tag) pinned = bool(t.core_pin) and not core_tag log(f"▶ {kind} „{tenant_id}“ · Version v{ver} · {image}{' (gepinnt)' if pinned else ''} · Modus: {'LIVE' if live else 'Dry-Run (keine Schreibzugriffe)'}") if not contract.supported(): raise DeployError(f"Core-Vertrag {contract.contract_version()} wird von dieser Admin-Version nicht unterstützt – Admin aktualisieren.") if source == "admin" and not bundle_ok and not dry: raise DeployError("Quelle „Admin“ ist noch gesperrt: Der Core kann das Bundle noch nicht laden (Einstellung core_supports_bundle).") if not dry and not live_enabled(): raise DeployError("Echte Deployments sind gesperrt (DEPLOY_LIVE != true).") if not dry and not guard.live_allowed(tenant_id): raise DeployError(guard.denial(tenant_id)) if live and source == "admin": with session() as db: url = setting(db, "admin_public_url") if not url.startswith("https://") or "<" in url: raise DeployError("Einstellung „Öffentliche URL der Admin-Plattform“ (https://…) fehlt – der Container könnte sein Bundle nicht holen.") if not kind.startswith("offboard"): # 1) OIDC-Client abgleichen (legt bei Bedarf an) – VOR dem Service, weil dessen Env die Client-Daten braucht rep = onyva_login.sync_tenant(tenant_id, apply=live, log=log, client=oidc_client, actor="deploy") if live and not rep["ok"]: raise DeployError(f"OIDC-Abgleich fehlgeschlagen: {rep['error']}") with session() as db: t = db.get(Tenant, tenant_id) if live: lacking = missing_secrets(db, t) if lacking: raise DeployError("Pflicht-Secrets ohne Wert: " + ", ".join(lacking) + " (Tenant → Secrets bzw. Domain & Login, geteilte unter Einstellungen).") definition = {"description": plan(db, t)["patch"]["description"], "image": image, "ports": [f"{PORT}:{PORT}/tcp"], "volumes": [f"vol-{t.id}:/data"], "environment": build_env(db, t, reveal=live)} mw = _client(live, log, http) sid = mw.resolve_stack_id(project, stack) if kind.startswith("offboard"): mw.offboard_service(stack_id=sid, service=f"app-{tenant_id}", volume=f"vol-{tenant_id}", project_id=project, hostname=host) log("Hinweis: Der OIDC-Client bleibt im Login bestehen (Tenant → Übersicht → Gefahrenbereich: „OIDC-Client im Login löschen“).") else: mw.deploy_service(stack_id=sid, service=f"app-{tenant_id}", volume=f"vol-{tenant_id}", definition=definition, container_port=PORT, project_id=project, hostname=host) warning = False if not dry: # Zustand sofort festhalten: geschrieben ist geschrieben, auch wenn der Health-Check noch wartet with session() as db: t = db.get(Tenant, tenant_id) if kind == "deploy": t.deployed_version, t.deployed_image, t.deployed_at = ver, tag, datetime.now(timezone.utc) store.audit(db, "system", f"{kind}.live", tenant_id, f"v{ver} :{tag}") if live and not kind.startswith("offboard") and host: fresh = mw.ingress_created # Neuer Ingress: Zertifikat/DNS bei Mittwald brauchen oft mehrere Minuten → geduldig warten (bis ~15 min) kw = {"tries": 90, "delay": 10.0} if fresh else {} if fresh: log("Neuer Ingress angelegt – Zertifikat/DNS bei Mittwald können einige Minuten dauern, Health-Check wartet geduldig …") try: args = (host, tag, ver if source == "admin" else None, log) health(*args) if health else verify_health(*args, **kw) except HealthTimeout as exc: if fresh and not exc.answered: # nur „noch nicht erreichbar“ bei neuem Ingress: Deploy ist durch, Prüfung später wiederholen warning = True log(f"⚠ {exc}") log("⚠ Container und Ingress sind angelegt, aber die Adresse antwortet noch nicht (Zertifikat/DNS?). " "Später „Health erneut prüfen“ (Deploy-Tab) oder „Läuft?“ (Übersicht).") else: raise log("✔ Fertig" if live and not warning else ("Fertig mit Warnung" if warning else "Dry-Run beendet – es wurde nichts geschrieben.")) return "warning" if warning else True except DeployError as exc: log(f"✘ {exc}") return False except Exception as exc: # noqa: BLE001 log(f"✘ Unerwarteter Fehler: {type(exc).__name__}: {exc}") return False def start(tenant_id: str, kind: str, actor: str, batch_id: str = "", http=None, wait: bool = False, oidc_client=None, core_tag: str | None = None, health=None) -> int: with session() as db: t = db.get(Tenant, tenant_id) d = Deployment(tenant_id=tenant_id, kind=kind, version=t.current_version, image_tag=effective_tag(db, t, core_tag), actor=actor, batch_id=batch_id, simulated=not (live_enabled() and kind in ("deploy", "offboard"))) db.add(d) db.flush() dep_id = d.id def work(): ok = execute(dep_id, tenant_id, kind, http, oidc_client, core_tag, health) _finish(dep_id, "warning" if ok == "warning" else ("success" if ok else "failed")) th = threading.Thread(target=work, daemon=True) th.start() if wait: th.join() return dep_id def run_sequence(tenant_ids: list[str], kind: str, actor: str, *, batch: str, core_tag: str | None = None, http=None, oidc_client=None, health=None, canary: list[str] | None = None, wait: bool = False, on_done=None) -> list[int]: """Tenants nacheinander (zuerst die Canary-Tenants), Abbruch beim ersten Fehler – der Rest wird „cancelled“. on_done(ok, results) wird nach dem Lauf aufgerufen (z. B. um ein Release abzuschließen).""" if kind in ("deploy", "offboard") and live_enabled(): # Live: nur freigegebene Tenants (siehe guard.py) skipped = [t for t in tenant_ids if not guard.live_allowed(t)] tenant_ids = [t for t in tenant_ids if guard.live_allowed(t)] if skipped: with session() as db: store.audit(db, actor, "deploy.skipped_not_allowed", None, ", ".join(skipped)) canary = [c for c in (canary or []) if c in tenant_ids] order = canary + [t for t in tenant_ids if t not in canary] with session() as db: rows = [] for tid in order: t = db.get(Tenant, tid) d = Deployment(tenant_id=tid, kind=kind, version=t.current_version, image_tag=effective_tag(db, t, core_tag), actor=actor, batch_id=batch, status="queued", simulated=not (live_enabled() and kind == "deploy")) db.add(d) rows.append(d) db.flush() dep_ids = [d.id for d in rows] def seq(): failed, results = False, {} for tid, did in zip(order, dep_ids): if failed: _append(did, "übersprungen – vorheriger Tenant fehlgeschlagen (Rollout abgebrochen)") _finish(did, "cancelled") results[tid] = "cancelled" continue with session() as db: db.get(Deployment, did).status = "running" ok = execute(did, tid, kind, http, oidc_client, core_tag, health) _finish(did, "warning" if ok == "warning" else ("success" if ok else "failed")) results[tid] = "warning" if ok == "warning" else ("success" if ok else "failed") failed = not ok if tid in canary and ok and tid == canary[-1]: _append(did, "Canary erfolgreich – restliche Tenants folgen") if on_done: on_done(not failed, results) th = threading.Thread(target=seq, daemon=True) th.start() if wait: th.join() return dep_ids def start_all(kind: str, actor: str, http=None, wait: bool = False, oidc_client=None, core_tag: str | None = None, health=None) -> tuple[str, list[int]]: """Alle aktiven, nicht gepinnten Tenants nacheinander; beim ersten Fehler Abbruch.""" batch = uuid.uuid4().hex[:10] with session() as db: ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status == "active").order_by(Tenant.id)) if not (core_tag and t.core_pin)] return batch, run_sequence(ids, kind, actor, batch=batch, core_tag=core_tag, http=http, oidc_client=oidc_client, health=health, wait=wait)