7d8c226da2
Admin-Plattform für die Tenants von Onyscribe: Editor mit Live-Vorschau (echter Core in Sandbox), Versionen, KI-Agent (Cortecs), OIDC-Abgleich über die Login-Developer-API, Deploy-Engine (Mittwald, gesperrt ohne DEPLOY_LIVE, Freigabe pro Tenant), Core-Vertrag (Schema/Manifest), Core-Releases mit Kompatibilitätsprüfung, Canary-Rollout und Rollback, Zustandsabgleich. Runbook: docs/RUNBOOK_PARALLELTEST.md Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
255 lines
13 KiB
Python
255 lines
13 KiB
Python
"""Core-Releases: Eingang (CI), Kompatibilitätsprüfung aller Tenants, Freigabe, Canary-Rollout, Rollback.
|
||
|
||
Ablauf: CI meldet Image-Tag → Admin (läuft auf genau diesem Core) prüft alle Tenants in der Sandbox → Freigabe →
|
||
Rollout (Canary zuerst, Abbruch beim ersten Fehler, Health-Check pro Tenant) → aktiver Tag wird der neue Standard.
|
||
Gepinnte Tenants nehmen nicht teil. Echte Rollouts brauchen DEPLOY_LIVE=true; sonst nur Dry-Run.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import io
|
||
import re
|
||
import threading
|
||
import uuid
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from datetime import datetime, timezone
|
||
|
||
from sqlalchemy import select
|
||
|
||
from app.db import session
|
||
from app.models import CoreRelease, Deployment, Setting, Tenant
|
||
from app.services import contract, deploy, guard, preview, store
|
||
|
||
DATE = re.compile(r"\b\d{1,2}\.\d{1,2}\.\d{2,4}\b")
|
||
|
||
|
||
class ReleaseError(RuntimeError):
|
||
pass
|
||
|
||
|
||
# ── Eingang ─────────────────────────────────────────────────────────────────
|
||
|
||
def announce(db, sha: str, tag: str | None, ref: str, image: str, message: str) -> tuple[CoreRelease, bool]:
|
||
if not re.fullmatch(r"[0-9a-fA-F]{7,64}", sha or ""):
|
||
raise ReleaseError("sha: 7–64 Hex-Zeichen erwartet")
|
||
sha = sha.lower()
|
||
r = db.scalar(select(CoreRelease).where(CoreRelease.sha == sha))
|
||
created = r is None
|
||
if created:
|
||
r = CoreRelease(sha=sha, tag=(tag or sha)[:100], ref=ref[:100], image=image[:300], message=message[:500])
|
||
db.add(r)
|
||
store.audit(db, "ci", "core.release", None, f"{sha[:10]} ({ref}) {message[:80]}")
|
||
else:
|
||
r.message, r.ref, r.image = message[:500] or r.message, ref[:100] or r.ref, image[:300] or r.image
|
||
db.flush()
|
||
return r, created
|
||
|
||
|
||
def admin_matches(r: CoreRelease) -> bool:
|
||
return contract.same_sha(contract.sha(), r.sha)
|
||
|
||
|
||
def view(r: CoreRelease, active_tag: str) -> dict:
|
||
return {"id": r.id, "sha": r.sha, "tag": r.tag, "ref": r.ref, "image": r.image, "message": r.message, "status": r.status,
|
||
"announced_at": r.announced_at.isoformat() if r.announced_at else None, "admin_matches": admin_matches(r),
|
||
"approved_by": r.approved_by, "compat": r.compat, "rollout_batch": r.rollout_batch, "previous_tag": r.previous_tag,
|
||
"active": r.tag == active_tag or contract.same_sha(r.sha, active_tag)}
|
||
|
||
|
||
# ── Kompatibilitätsprüfung ──────────────────────────────────────────────────
|
||
|
||
def _pdf_fingerprint(pdf: bytes) -> tuple[int, str]:
|
||
from pypdf import PdfReader
|
||
rd = PdfReader(io.BytesIO(pdf))
|
||
text = DATE.sub("<datum>", "\n".join((p.extract_text() or "") for p in rd.pages)) # das Tagesdatum steht im PDF
|
||
return len(rd.pages), hashlib.sha256(text.encode()).hexdigest()
|
||
|
||
|
||
def fingerprint(db, t: Tenant) -> dict:
|
||
"""Rendert den veröffentlichten Stand mit dem aktuellen Admin-Core und bildet Fingerabdrücke."""
|
||
cfg, files = store.content(db, t, "current")
|
||
sample = preview.sample_for(t)
|
||
fp: dict = {"sample": hashlib.sha256(sample.encode()).hexdigest()[:12]}
|
||
val = preview.run_content(db, t, cfg, files, "validate", sample)
|
||
fp["validate"] = {"ok": val.get("ok", False), "errors": val.get("errors", []), "warnings": val.get("warnings", [])}
|
||
if val.get("ok"):
|
||
pages, text_sha = _pdf_fingerprint(preview.run_content(db, t, cfg, files, "pdf", sample))
|
||
fp.update(pdf_pages=pages, pdf_text=text_sha,
|
||
web=hashlib.sha256(preview.run_content(db, t, cfg, files, "index", sample)).hexdigest(),
|
||
prompt=hashlib.sha256(preview.run_content(db, t, cfg, files, "prompt", sample, {"title": "x"})["prompt"].encode()).hexdigest())
|
||
fp["schema_problems"] = contract.check_tenant_json(cfg)
|
||
return fp
|
||
|
||
|
||
def compare(fp: dict, baseline: dict | None, version: int) -> dict:
|
||
"""→ {status: ok|changed|new|error, changed:[…], errors:[…], warnings:[…]}"""
|
||
out = {"status": "ok", "changed": [], "errors": fp["validate"]["errors"], "warnings": list(fp["validate"]["warnings"]) + fp.get("schema_problems", [])}
|
||
if not fp["validate"]["ok"]:
|
||
out["status"] = "error"
|
||
return out
|
||
if not baseline or baseline.get("version") != version or baseline["fp"].get("sample") != fp["sample"]:
|
||
out["status"] = "new"
|
||
return out
|
||
labels = {"pdf_pages": "PDF-Seitenzahl", "pdf_text": "PDF-Text", "web": "Web-UI", "prompt": "Finaler Prompt"}
|
||
out["changed"] = [labels[k] for k in labels if baseline["fp"].get(k) != fp.get(k)]
|
||
if out["changed"]:
|
||
out["status"] = "changed"
|
||
return out
|
||
|
||
|
||
def check(release_id: int, *, force: bool = False, wait: bool = False, workers: int = 3) -> None:
|
||
"""Prüft ALLE Tenants (außer archivierten) mit dem Core, auf dem die Admin-Plattform läuft. Ergebnis in release.compat."""
|
||
with session() as db:
|
||
r = db.get(CoreRelease, release_id)
|
||
if r is None:
|
||
raise ReleaseError("Release nicht gefunden")
|
||
if not force and not admin_matches(r):
|
||
raise ReleaseError(f"Die Admin-Plattform läuft noch auf Core {contract.sha()[:10]}, das Release ist {r.sha[:10]}. "
|
||
"Erst den Admin neu bauen (FROM neues Core-Image) – sonst würde gegen den falschen Core geprüft.")
|
||
ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status != "archived").order_by(Tenant.id))]
|
||
r.compat = {"checked_at": None, "core_sha": contract.sha(), "running": True, "tenants": {}, "summary": {}, "forced": force and not admin_matches(r)}
|
||
r.status = "new" if r.status in ("new", "checked") else r.status
|
||
|
||
def one(tid: str):
|
||
with session() as db:
|
||
t = db.get(Tenant, tid)
|
||
try:
|
||
fp = fingerprint(db, t)
|
||
res = compare(fp, t.render_baseline, t.current_version)
|
||
except Exception as exc: # noqa: BLE001
|
||
fp, res = {}, {"status": "error", "changed": [], "errors": [f"{type(exc).__name__}: {str(exc)[-300:]}"], "warnings": []}
|
||
return tid, fp, res
|
||
|
||
def work():
|
||
results = {}
|
||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||
for tid, fp, res in ex.map(one, ids):
|
||
results[tid] = {**res, "fp": fp}
|
||
summary = {k: sum(1 for v in results.values() if v["status"] == k) for k in ("ok", "changed", "new", "error")}
|
||
with session() as db:
|
||
r = db.get(CoreRelease, release_id)
|
||
r.compat = {"checked_at": datetime.now(timezone.utc).isoformat(), "core_sha": contract.sha(), "running": False, "tenants": results,
|
||
"summary": summary, "forced": bool(r.compat and r.compat.get("forced"))}
|
||
if r.status == "new":
|
||
r.status = "checked"
|
||
store.audit(db, "system", "core.check", None, f"{r.sha[:10]}: {summary}")
|
||
|
||
th = threading.Thread(target=work, daemon=True)
|
||
th.start()
|
||
if wait:
|
||
th.join()
|
||
|
||
|
||
# ── Freigabe & Rollout ──────────────────────────────────────────────────────
|
||
|
||
def approve(db, release_id: int, actor: str, acknowledge: bool = False) -> CoreRelease:
|
||
r = db.get(CoreRelease, release_id)
|
||
if r is None:
|
||
raise ReleaseError("Release nicht gefunden")
|
||
c = r.compat
|
||
if not c or c.get("running"):
|
||
raise ReleaseError("Erst die Kompatibilitätsprüfung abschließen.")
|
||
if not c.get("forced") and not contract.same_sha(c.get("core_sha", ""), r.sha):
|
||
raise ReleaseError("Die Prüfung stammt von einem anderen Core als dieses Release – bitte erneut prüfen.")
|
||
if c["summary"].get("error"):
|
||
bad = [t for t, v in c["tenants"].items() if v["status"] == "error"]
|
||
raise ReleaseError(f"Nicht freigebbar: {len(bad)} Tenant(s) mit Fehlern ({', '.join(bad)}).")
|
||
if c["summary"].get("changed") and not acknowledge:
|
||
raise ReleaseError("Bei einigen Tenants hat sich das Rendering geändert – bitte prüfen und bewusst bestätigen (acknowledge).")
|
||
r.status, r.approved_by, r.approved_at = "approved", actor, datetime.now(timezone.utc)
|
||
store.audit(db, actor, "core.approve", None, r.sha[:10])
|
||
return r
|
||
|
||
|
||
def canary_list(db) -> list[str]:
|
||
s = db.get(Setting, "canary_tenants")
|
||
return [x.strip() for x in (s.value if s and s.value else "demo").split(",") if x.strip()]
|
||
|
||
|
||
def _set_setting(db, key: str, value: str) -> None:
|
||
s = db.get(Setting, key)
|
||
if s is None:
|
||
s = Setting(key=key)
|
||
db.add(s)
|
||
s.value = value
|
||
|
||
|
||
def rollout(release_id: int, actor: str, *, dry_run: bool, http=None, oidc_client=None, health=None, wait: bool = False) -> tuple[str, list[int]]:
|
||
live = not dry_run
|
||
if live and not deploy.live_enabled():
|
||
raise ReleaseError("Echte Rollouts sind gesperrt (DEPLOY_LIVE != true). Dry-Run ist möglich.")
|
||
with session() as db:
|
||
r = db.get(CoreRelease, release_id)
|
||
if r is None:
|
||
raise ReleaseError("Release nicht gefunden")
|
||
allowed = {"approved", "active", "failed", "rolled_back"} | ({"new", "checked"} if dry_run else set()) # Dry-Run schon vor der Freigabe
|
||
if r.status not in allowed:
|
||
raise ReleaseError(f"Release ist im Status „{r.status}“ – erst prüfen und freigeben.")
|
||
tag, sha = r.tag, r.sha
|
||
ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status == "active").order_by(Tenant.id)) if not t.core_pin]
|
||
if live: # Live: nur für Live-Schreibzugriffe freigegebene Tenants (guard.py)
|
||
ids = [t for t in ids if guard.live_allowed(t)]
|
||
if not ids:
|
||
raise ReleaseError(f"Kein Tenant für Live-Rollouts freigegeben (DEPLOY_ALLOWED_TENANTS = {guard.describe()}).")
|
||
canary = canary_list(db)
|
||
batch = uuid.uuid4().hex[:10]
|
||
if live:
|
||
r.status, r.previous_tag, r.rollout_batch = "rolling", deploy.setting(db, "core_image_tag", "latest"), batch
|
||
store.audit(db, actor, "core.rollout", None, f"{sha[:10]} → {len(ids)} Tenants")
|
||
|
||
def done(ok: bool, results: dict):
|
||
if not live:
|
||
return
|
||
with session() as db:
|
||
r = db.get(CoreRelease, release_id)
|
||
if ok:
|
||
r.status = "active"
|
||
_set_setting(db, "core_image_tag", tag)
|
||
for other in db.scalars(select(CoreRelease).where(CoreRelease.status == "active", CoreRelease.id != release_id)):
|
||
other.status = "superseded"
|
||
for tid in results: # Baselines für künftige Kompatibilitätsprüfungen
|
||
t = db.get(Tenant, tid)
|
||
c = ((r.compat or {}).get("tenants") or {}).get(tid)
|
||
if t and c and c.get("fp"):
|
||
t.render_baseline = {"version": t.current_version, "core": sha, "fp": c["fp"]}
|
||
else:
|
||
r.status = "failed"
|
||
store.audit(db, "system", "core.rollout.done", None, f"{sha[:10]} {'ok' if ok else 'FEHLER'} {results}")
|
||
|
||
dep_ids = deploy.run_sequence(ids, "dry-run" if dry_run else "deploy", actor, batch=batch, core_tag=tag, http=http, oidc_client=oidc_client,
|
||
health=health, canary=canary, wait=wait, on_done=done)
|
||
return batch, dep_ids
|
||
|
||
|
||
def rollback_release(release_id: int, actor: str, *, http=None, oidc_client=None, health=None, wait: bool = False) -> tuple[str, list[int]]:
|
||
"""Alle (nicht gepinnten) Tenants auf den Tag vor diesem Rollout zurück."""
|
||
if not deploy.live_enabled():
|
||
raise ReleaseError("Echte Rollbacks sind gesperrt (DEPLOY_LIVE != true).")
|
||
with session() as db:
|
||
r = db.get(CoreRelease, release_id)
|
||
if r is None or not r.previous_tag:
|
||
raise ReleaseError("Kein vorheriger Tag bekannt – Rollback nicht möglich.")
|
||
prev = r.previous_tag
|
||
ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status == "active").order_by(Tenant.id)) if not t.core_pin and guard.live_allowed(t.id)]
|
||
batch = uuid.uuid4().hex[:10]
|
||
store.audit(db, actor, "core.rollback", None, f"{r.sha[:10]} → :{prev}")
|
||
|
||
def done(ok, results):
|
||
with session() as db:
|
||
r = db.get(CoreRelease, release_id)
|
||
if ok:
|
||
r.status = "rolled_back"
|
||
_set_setting(db, "core_image_tag", prev)
|
||
store.audit(db, "system", "core.rollback.done", None, f"{'ok' if ok else 'FEHLER'} {results}")
|
||
|
||
return batch, deploy.run_sequence(ids, "deploy", actor, batch=batch, core_tag=prev, http=http, oidc_client=oidc_client, health=health, wait=wait, on_done=done)
|
||
|
||
|
||
def tenant_rollback_tag(db, t: Tenant) -> str | None:
|
||
"""Letzter erfolgreich live deployter Tag dieses Tenants, der vom aktuellen abweicht."""
|
||
for d in db.scalars(select(Deployment).where(Deployment.tenant_id == t.id, Deployment.kind == "deploy", Deployment.status == "success",
|
||
Deployment.simulated.is_(False)).order_by(Deployment.id.desc())):
|
||
if d.image_tag and d.image_tag != t.deployed_image:
|
||
return d.image_tag
|
||
return None
|