Onyscribe Admin: Initialer Stand
Admin-Plattform für die Tenants von Onyscribe: Editor mit Live-Vorschau (echter Core in Sandbox), Versionen, KI-Agent (Cortecs), OIDC-Abgleich über die Login-Developer-API, Deploy-Engine (Mittwald, gesperrt ohne DEPLOY_LIVE, Freigabe pro Tenant), Core-Vertrag (Schema/Manifest), Core-Releases mit Kompatibilitätsprüfung, Canary-Rollout und Rollback, Zustandsabgleich. Runbook: docs/RUNBOOK_PARALLELTEST.md Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
RubenRWU
0
app/services/__init__.py
Normal file
0
app/services/__init__.py
Normal file
125
app/services/contract.py
Normal file
125
app/services/contract.py
Normal file
@@ -0,0 +1,125 @@
|
||||
"""Vertrag mit dem Core (siehe transkript-platform/docs/CONTRACT.md): Schema, Manifest, Build-Info, Env-Spezifikation.
|
||||
|
||||
Gelesen wird aus dem Core, auf dem die Admin-Plattform läuft (PLATFORM_DIR: im Container das Core-Image, lokal der vendored Stand).
|
||||
`core/contract.py` ist import-sicher (nur Standardbibliothek) und wird per Dateipfad geladen – der Core-Code selbst wird NIE
|
||||
in diesem Prozess importiert (er hält Prozess-Singletons); gerendert wird in der Sandbox.
|
||||
Ältere Cores ohne Vertrag („legacy“) funktionieren weiter: Schema fehlt → feste Admin-Formulare und feste Env-Liste.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from app.settings import PLATFORM_DIR
|
||||
|
||||
SUPPORTED_CONTRACT = {1}
|
||||
|
||||
# Rückfall für Cores ohne Schema (entspricht dem Stand vor dem Vertrag)
|
||||
LEGACY_SHARED_ENV = {
|
||||
"OIDC_ISSUER": "oidc_issuer", "AUTH_ENABLED": "auth_enabled", "APP_COOKIE_SECURE": "app_cookie_secure",
|
||||
"MAX_FILE_SIZE_MB": "max_file_size_mb", "LLM_MODEL": "llm_model", "LLM_MAX_TOKENS": "llm_max_tokens",
|
||||
"LLM_TEMPERATURE": "llm_temperature", "CORTECS_BASE_URL": "cortecs_base_url", "SMTP_HOST": "smtp_host",
|
||||
"SMTP_PORT": "smtp_port", "SMTP_USER": "smtp_user", "SMTP_FROM": "smtp_from",
|
||||
"MISTRAL_TRANSCRIBE_MODEL": "mistral_transcribe_model", "MISTRAL_DIARIZE": "mistral_diarize",
|
||||
"MISTRAL_TIMEOUT": "mistral_timeout", "COST_API_BASE": "cost_api_base", "COST_TRACKING_ENABLED": "cost_tracking_enabled",
|
||||
}
|
||||
LEGACY_REQUIRED_SECRETS = ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET"]
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _module():
|
||||
path = Path(PLATFORM_DIR) / "core" / "contract.py"
|
||||
if not path.exists():
|
||||
return None
|
||||
spec = importlib.util.spec_from_file_location("onyscribe_core_contract", path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def reload() -> None:
|
||||
_module.cache_clear()
|
||||
schema.cache_clear()
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
return _module() is not None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def schema() -> dict | None:
|
||||
m = _module()
|
||||
return m.load_schema() if m else None
|
||||
|
||||
|
||||
def build_info() -> dict:
|
||||
m = _module()
|
||||
if m:
|
||||
return m.build_info()
|
||||
f = Path(PLATFORM_DIR) / "CORE_VERSION"
|
||||
return {"sha": f.read_text().split()[0] if f.exists() else "unbekannt", "built_at": None, "source": "legacy"}
|
||||
|
||||
|
||||
def manifest() -> dict:
|
||||
m = _module()
|
||||
if m:
|
||||
return m.build_manifest()
|
||||
return {"contract": 0, "core": build_info(), "legacy": True}
|
||||
|
||||
|
||||
def contract_version() -> int:
|
||||
return int(manifest().get("contract", 0))
|
||||
|
||||
|
||||
def supported() -> bool:
|
||||
return (not available()) or contract_version() in SUPPORTED_CONTRACT
|
||||
|
||||
|
||||
def sha() -> str:
|
||||
return build_info()["sha"]
|
||||
|
||||
|
||||
def same_sha(a: str, b: str) -> bool:
|
||||
a, b = (a or "").lower(), (b or "").lower()
|
||||
return bool(a and b) and (a.startswith(b) or b.startswith(a)) and min(len(a), len(b)) >= 7
|
||||
|
||||
|
||||
def check_tenant_json(raw: dict) -> list[str]:
|
||||
m = _module()
|
||||
return m.check_tenant_json(raw, schema()) if m else []
|
||||
|
||||
|
||||
def field_paths() -> set[str]:
|
||||
s = schema()
|
||||
return {f["path"] for f in s["fields"]} if s else set()
|
||||
|
||||
|
||||
def env_spec() -> list[dict]:
|
||||
s = schema()
|
||||
if s:
|
||||
return s["env"]
|
||||
return [{"name": n, "kind": "setting", "scope": "shared", "setting": k, "default": None, "label": n, "group": "Allgemein"} for n, k in LEGACY_SHARED_ENV.items()]
|
||||
|
||||
|
||||
def required_secrets(tenant_cfg: dict | None = None) -> list[str]:
|
||||
"""Pflicht-Secrets laut Schema (required, required_if wie „stt.provider!=cortecs“ gegen die Tenant-Config ausgewertet)."""
|
||||
s = schema()
|
||||
if not s:
|
||||
return list(LEGACY_REQUIRED_SECRETS)
|
||||
out = []
|
||||
for e in s["env"]:
|
||||
if e["kind"] != "secret":
|
||||
continue
|
||||
if e.get("required"):
|
||||
out.append(e["name"])
|
||||
elif e.get("required_if"):
|
||||
path, _, rhs = e["required_if"].partition("!=")
|
||||
cur = tenant_cfg or {}
|
||||
for k in path.split("."):
|
||||
cur = cur.get(k, {}) if isinstance(cur, dict) else {}
|
||||
val = cur if isinstance(cur, str) else ""
|
||||
if (val or "mistral") != rhs: # Provider-Standard ist mistral
|
||||
out.append(e["name"])
|
||||
return out
|
||||
15
app/services/crypto.py
Normal file
15
app/services/crypto.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from app.settings import SECRET_KEY
|
||||
|
||||
_f = Fernet(SECRET_KEY.encode())
|
||||
|
||||
|
||||
def encrypt(v: str) -> str:
|
||||
return _f.encrypt(v.encode()).decode()
|
||||
|
||||
|
||||
def decrypt(v: str) -> str:
|
||||
return _f.decrypt(v.encode()).decode()
|
||||
44
app/services/defaults.py
Normal file
44
app/services/defaults.py
Normal file
@@ -0,0 +1,44 @@
|
||||
import json
|
||||
import secrets
|
||||
|
||||
# Standardfarben des Cores (core/config.py DEFAULT_COLORS) – Basis für neue Tenants
|
||||
DEFAULT_COLORS = {
|
||||
"accent": "#327F24", "accent_dark": "#235A19", "accent_mid": "#3E9130", "accent_light": "#EAF4E7",
|
||||
"accent_xlight": "#F3FAF1", "bg": "#F1F5F0", "surface": "#FFFFFF", "surface2": "#F6FAF5",
|
||||
"border": "#DCE8D9", "border2": "#BDD1B8", "text": "#444444", "text_2": "#666666", "text_3": "#9AA89A",
|
||||
"red": "#DC2626", "progress_to": "#8bbf43",
|
||||
}
|
||||
|
||||
COLOR_LABELS = {
|
||||
"accent": "Akzent", "accent_dark": "Akzent dunkel", "accent_mid": "Akzent mittel", "accent_light": "Akzent hell",
|
||||
"accent_xlight": "Akzent extra hell", "bg": "Seitenhintergrund", "surface": "Karten", "surface2": "Karten 2",
|
||||
"border": "Rahmen", "border2": "Rahmen 2", "text": "Text", "text_2": "Text sekundär", "text_3": "Text tertiär",
|
||||
"red": "Rot (Hinweis)", "progress_to": "Fortschritt (Verlauf-Ende)",
|
||||
}
|
||||
|
||||
# Secret-Schlüssel = Env-Namen im Container (siehe infra/deploy.py)
|
||||
REQUIRED_SECRETS = ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "COST_INGEST_KEY"]
|
||||
OPTIONAL_SECRETS = ["MISTRAL_API_KEY"]
|
||||
SECRET_HELP = {
|
||||
"OIDC_CLIENT_ID": "Client-ID aus login.onyva.dev (manuell angelegt)",
|
||||
"OIDC_CLIENT_SECRET": "Client-Secret aus login.onyva.dev",
|
||||
"SESSION_SECRET": "Signiert Session-Cookies – kann hier erzeugt werden",
|
||||
"COST_INGEST_KEY": "Schlüssel für die Kostenerfassung (Cost-API)",
|
||||
"MISTRAL_API_KEY": "Nur nötig bei STT-Provider „mistral“ (bei „cortecs“ wird der geteilte Key genutzt)",
|
||||
"ADMIN_BUNDLE_TOKEN": "Token, mit dem der Container sein Bundle bei der Admin-Plattform abholt",
|
||||
}
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(48)
|
||||
|
||||
|
||||
def skeleton_config(tenant_id: str, name: str) -> dict:
|
||||
return {
|
||||
"id": tenant_id, "name": name, "title": f"{name} – Gesprächsprotokoll",
|
||||
"description": f"Gesprächsprotokoll-Tool für {name}", "ai_detect_org": name,
|
||||
"org_short": name, "org_profile": "",
|
||||
"flags": {}, "stt": {"provider": "cortecs", "model": "voxtral-mini-transcribe-2602"},
|
||||
"colors": dict(DEFAULT_COLORS),
|
||||
"office": {"name": name, "subtitle": "", "address": "", "tel": "", "email": "", "website": ""},
|
||||
}
|
||||
314
app/services/deploy.py
Normal file
314
app/services/deploy.py
Normal file
@@ -0,0 +1,314 @@
|
||||
"""Deploy-Orchestrierung: Plan (wie infra/deploy.py), Ausführung über den Mittwald-Client.
|
||||
|
||||
Sicherheit: Ohne `DEPLOY_LIVE=true` (Env der Admin-Plattform) ist JEDE schreibende Operation gesperrt – es gibt nur Dry-Runs.
|
||||
Außerdem blockiert der Core-Kompatibilitäts-Schalter (`core_supports_bundle`) Tenants mit Quelle „Admin“,
|
||||
solange der Core das Bundle noch nicht laden kann.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import session
|
||||
from app.models import Deployment, Secret, Setting, Tenant
|
||||
from app.services import contract, crypto, guard, onyva_login, store
|
||||
from app.services.mittwald import DeployError, Mittwald, mask_env
|
||||
|
||||
PORT = "8276"
|
||||
|
||||
|
||||
def live_enabled() -> bool:
|
||||
return os.getenv("DEPLOY_LIVE", "false").lower() == "true"
|
||||
|
||||
|
||||
def token_present() -> bool:
|
||||
return bool(os.getenv("MITTWALD_API_TOKEN", "").strip())
|
||||
|
||||
|
||||
def setting(db, key: str, default: str = "") -> str:
|
||||
s = db.get(Setting, key)
|
||||
return s.value if s and s.value != "" else default
|
||||
|
||||
|
||||
def host_for(db, t: Tenant) -> str:
|
||||
if t.custom_domain and t.domain_status == "verified":
|
||||
return t.custom_domain
|
||||
return f"{t.subdomain or t.id}.{setting(db, 'base_domain', 'example.de')}"
|
||||
|
||||
|
||||
def effective_tag(db, t: Tenant | None = None, override: str | None = None) -> str:
|
||||
"""Welcher Core-Tag gilt? Override (Rollout) > Pin des Tenants > freigegebener Standard-Tag."""
|
||||
return override or (t.core_pin if t is not None and t.core_pin else None) or setting(db, "core_image_tag", "latest")
|
||||
|
||||
|
||||
def image_ref(db, t: Tenant | None = None, tag: str | None = None) -> str:
|
||||
registry = setting(db, "registry", "").rstrip("/")
|
||||
name = setting(db, "image_name", "transkript-platform")
|
||||
base = f"{registry}/{name}" if registry else name
|
||||
return f"{base.lower()}:{effective_tag(db, t, tag)}"
|
||||
|
||||
|
||||
def build_env(db, t: Tenant, reveal: bool) -> dict[str, str]:
|
||||
"""Container-Env, abgeleitet aus dem Env-Schema des Cores (kind/scope) – neue Core-Variablen erscheinen ohne Admin-Änderung.
|
||||
reveal=False → Secrets als Platzhalter."""
|
||||
host = host_for(db, t)
|
||||
env = {"TENANT": t.id, "DATA_DIR": "/data", "PORT": PORT,
|
||||
"OIDC_REDIRECT_URI": f"https://{host}/auth/callback", "OIDC_LOGOUT_REDIRECT_URI": f"https://{host}/"}
|
||||
spec = contract.env_spec()
|
||||
for e in spec: # geteilte, nicht geheime Einstellungen
|
||||
if e["kind"] == "setting" and e["scope"] == "shared":
|
||||
v = setting(db, e.get("setting") or e["name"].lower())
|
||||
if v:
|
||||
env[e["name"]] = v
|
||||
shared = db.scalars(select(Secret).where(Secret.tenant_id.is_(None))).all()
|
||||
own = db.scalars(select(Secret).where(Secret.tenant_id == t.id)).all()
|
||||
for s_ in [*shared, *own]: # Tenant-Werte überschreiben geteilte; auch Secrets ohne Schema-Eintrag werden durchgereicht
|
||||
if s_.key == "ADMIN_BUNDLE_TOKEN" and t.source != "admin":
|
||||
continue
|
||||
if s_.value_enc:
|
||||
env[s_.key] = crypto.decrypt(s_.value_enc) if reveal else "•••• gesetzt"
|
||||
elif not reveal:
|
||||
env[s_.key] = "⚠ fehlt (Wert noch nicht in der Admin-DB)"
|
||||
if t.source == "admin":
|
||||
env["TENANT_SOURCE"] = "admin"
|
||||
env["ADMIN_URL"] = setting(db, "admin_public_url", "https://<admin-host>")
|
||||
return env
|
||||
|
||||
|
||||
def missing_secrets(db, t: Tenant) -> list[str]:
|
||||
"""Pflicht-Secrets laut Schema, die weder beim Tenant noch geteilt einen Wert haben."""
|
||||
cfg, _ = store.content(db, t, "current")
|
||||
have = {s.key for s in db.scalars(select(Secret).where((Secret.tenant_id == t.id) | Secret.tenant_id.is_(None))) if s.value_enc}
|
||||
need = set(contract.required_secrets(cfg))
|
||||
if t.source == "admin":
|
||||
need.add("ADMIN_BUNDLE_TOKEN")
|
||||
return sorted(need - have)
|
||||
|
||||
|
||||
def plan(db, t: Tenant) -> dict:
|
||||
cfg, _ = store.content(db, t, "current")
|
||||
return {
|
||||
"service": f"app-{t.id}", "volume": f"vol-{t.id}",
|
||||
"patch": {"description": cfg.get("name") or f"Onyscribe {t.id}", "image": image_ref(db, t),
|
||||
"environment": build_env(db, t, reveal=False), "ports": [f"{PORT}:{PORT}/tcp"], "volumes": [f"vol-{t.id}:/data"]},
|
||||
"ingress": {"host": host_for(db, t), "target": f"app-{t.id}:{PORT}"},
|
||||
"missing_secrets": missing_secrets(db, t),
|
||||
"oidc": {"mode": onyva_login.config_of(t)["mode"], "key": onyva_login.key_present(), "writes": onyva_login.writes_allowed(),
|
||||
"last_sync": onyva_login.config_of(t)["last_sync"]},
|
||||
"mode": {"live": live_enabled(), "token": token_present(), "tenant_allowed": guard.live_allowed(t.id), "allowed": guard.describe(), "project": bool(setting(db, "mittwald_project_id")),
|
||||
"core_bundle": setting(db, "core_supports_bundle", "false") == "true"},
|
||||
"core": {"tag": effective_tag(db, t), "pinned": bool(t.core_pin), "admin_core": contract.sha()[:10], "contract": contract.contract_version()},
|
||||
}
|
||||
|
||||
|
||||
# ── Ausführung ──────────────────────────────────────────────────────────────
|
||||
|
||||
def _append(dep_id: int, line: str) -> None:
|
||||
with session() as db:
|
||||
db.get(Deployment, dep_id).log += f"{datetime.now(timezone.utc).strftime('%H:%M:%S')} {line}\n"
|
||||
|
||||
|
||||
def _finish(dep_id: int, status: str) -> None:
|
||||
with session() as db:
|
||||
d = db.get(Deployment, dep_id)
|
||||
d.status, d.finished_at = status, datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _client(live: bool, log, http=None) -> Mittwald:
|
||||
return Mittwald(os.getenv("MITTWALD_API_TOKEN", "").strip(), os.getenv("MITTWALD_API_BASE", "https://api.mittwald.de/v2"), live, log, http)
|
||||
|
||||
|
||||
def _hex_tag(tag: str) -> bool:
|
||||
return bool(tag) and all(c in "0123456789abcdef" for c in tag.lower()) and len(tag) >= 7
|
||||
|
||||
|
||||
def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, log, *, get=None, tries: int = 18, delay: float = 5.0,
|
||||
sleep=time.sleep) -> None:
|
||||
"""Prüft nach dem Deploy, ob wirklich läuft, was wir ausgerollt haben (Vertrag: /health.core / .bundle).
|
||||
Wirft DeployError, wenn der Container nach `tries` Versuchen nicht gesund ist oder die falsche Version meldet."""
|
||||
if get is None:
|
||||
import httpx
|
||||
|
||||
def get(url):
|
||||
r = httpx.get(url, timeout=8, follow_redirects=True)
|
||||
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
|
||||
last = "keine Antwort"
|
||||
for i in range(tries):
|
||||
try:
|
||||
status, body = get(f"https://{host}/health")
|
||||
problems = []
|
||||
if status != 200 or body.get("status") != "ok":
|
||||
problems.append(f"HTTP {status}")
|
||||
core = body.get("core")
|
||||
if expect_sha and _hex_tag(expect_sha):
|
||||
if core is None:
|
||||
log("⚠ /health meldet keine Core-Version (Image vor dem Vertrag) – Version nicht prüfbar")
|
||||
elif not contract.same_sha(core.get("version", ""), expect_sha):
|
||||
problems.append(f"Core {str(core.get('version'))[:10]} statt {expect_sha[:10]}")
|
||||
if expect_bundle is not None and body.get("bundle"):
|
||||
b = body["bundle"]
|
||||
if b.get("source") != "admin" or b.get("version") != expect_bundle or b.get("stale"):
|
||||
problems.append(f"Bundle {b.get('source')} v{b.get('version')}{' (veraltet)' if b.get('stale') else ''} statt admin v{expect_bundle}"
|
||||
+ (f" – {b.get('error')}" if b.get("error") else ""))
|
||||
if not problems:
|
||||
log(f"Health-Check https://{host}/health ✔" + (f" · Core {core['version'][:10]}" if core else "")
|
||||
+ (f" · Bundle v{body['bundle']['version']}" if body.get("bundle", {}).get("source") == "admin" else ""))
|
||||
return
|
||||
last = "; ".join(problems)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last = f"{type(exc).__name__}: {exc}"
|
||||
if i < tries - 1:
|
||||
sleep(delay)
|
||||
raise DeployError(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}")
|
||||
|
||||
|
||||
def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None) -> bool:
|
||||
"""Führt einen Deployment-Datensatz aus. kind: dry-run | deploy | offboard-dry | offboard.
|
||||
core_tag überschreibt den Core-Tag (Rollout/Rollback); `health(host, expect_sha, expect_bundle, log)` ist in Tests ersetzbar."""
|
||||
log = lambda m: _append(dep_id, m) # noqa: E731
|
||||
dry = kind in ("dry-run", "offboard-dry")
|
||||
live = live_enabled() and not dry
|
||||
try:
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tenant_id)
|
||||
ver = t.current_version
|
||||
project, stack = setting(db, "mittwald_project_id"), setting(db, "mittwald_stack_id")
|
||||
host = host_for(db, t) if setting(db, "base_domain") else None
|
||||
source, bundle_ok = t.source, setting(db, "core_supports_bundle", "false") == "true"
|
||||
image, tag = image_ref(db, t, core_tag), effective_tag(db, t, core_tag)
|
||||
pinned = bool(t.core_pin) and not core_tag
|
||||
log(f"▶ {kind} „{tenant_id}“ · Version v{ver} · {image}{' (gepinnt)' if pinned else ''} · Modus: {'LIVE' if live else 'Dry-Run (keine Schreibzugriffe)'}")
|
||||
if not contract.supported():
|
||||
raise DeployError(f"Core-Vertrag {contract.contract_version()} wird von dieser Admin-Version nicht unterstützt – Admin aktualisieren.")
|
||||
if source == "admin" and not bundle_ok and not dry:
|
||||
raise DeployError("Quelle „Admin“ ist noch gesperrt: Der Core kann das Bundle noch nicht laden (Einstellung core_supports_bundle).")
|
||||
if not dry and not live_enabled():
|
||||
raise DeployError("Echte Deployments sind gesperrt (DEPLOY_LIVE != true).")
|
||||
if not dry and not guard.live_allowed(tenant_id):
|
||||
raise DeployError(guard.denial(tenant_id))
|
||||
if live and source == "admin":
|
||||
with session() as db:
|
||||
url = setting(db, "admin_public_url")
|
||||
if not url.startswith("https://") or "<" in url:
|
||||
raise DeployError("Einstellung „Öffentliche URL der Admin-Plattform“ (https://…) fehlt – der Container könnte sein Bundle nicht holen.")
|
||||
|
||||
if not kind.startswith("offboard"):
|
||||
# 1) OIDC-Client abgleichen (legt bei Bedarf an) – VOR dem Service, weil dessen Env die Client-Daten braucht
|
||||
rep = onyva_login.sync_tenant(tenant_id, apply=live, log=log, client=oidc_client, actor="deploy")
|
||||
if live and not rep["ok"]:
|
||||
raise DeployError(f"OIDC-Abgleich fehlgeschlagen: {rep['error']}")
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tenant_id)
|
||||
if live:
|
||||
lacking = missing_secrets(db, t)
|
||||
if lacking:
|
||||
raise DeployError("Pflicht-Secrets ohne Wert: " + ", ".join(lacking) + " (Tenant → Secrets bzw. Domain & Login, geteilte unter Einstellungen).")
|
||||
definition = {"description": plan(db, t)["patch"]["description"], "image": image, "ports": [f"{PORT}:{PORT}/tcp"],
|
||||
"volumes": [f"vol-{t.id}:/data"], "environment": build_env(db, t, reveal=live)}
|
||||
mw = _client(live, log, http)
|
||||
sid = mw.resolve_stack_id(project, stack)
|
||||
if kind.startswith("offboard"):
|
||||
mw.offboard_service(stack_id=sid, service=f"app-{tenant_id}", volume=f"vol-{tenant_id}", project_id=project, hostname=host)
|
||||
log("Hinweis: Der OIDC-Client bleibt im Login bestehen (Tenant → Übersicht → Gefahrenbereich: „OIDC-Client im Login löschen“).")
|
||||
else:
|
||||
mw.deploy_service(stack_id=sid, service=f"app-{tenant_id}", volume=f"vol-{tenant_id}", definition=definition,
|
||||
container_port=PORT, project_id=project, hostname=host)
|
||||
if live and not kind.startswith("offboard") and host:
|
||||
(health or verify_health)(host, tag, ver if source == "admin" else None, log)
|
||||
if not dry:
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tenant_id)
|
||||
if kind == "deploy":
|
||||
t.deployed_version, t.deployed_image, t.deployed_at = ver, tag, datetime.now(timezone.utc)
|
||||
store.audit(db, "system", f"{kind}.live", tenant_id, f"v{ver} :{tag}")
|
||||
log("✔ Fertig" if live else "Dry-Run beendet – es wurde nichts geschrieben.")
|
||||
return True
|
||||
except DeployError as exc:
|
||||
log(f"✘ {exc}")
|
||||
return False
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log(f"✘ Unerwarteter Fehler: {type(exc).__name__}: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def start(tenant_id: str, kind: str, actor: str, batch_id: str = "", http=None, wait: bool = False, oidc_client=None,
|
||||
core_tag: str | None = None, health=None) -> int:
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tenant_id)
|
||||
d = Deployment(tenant_id=tenant_id, kind=kind, version=t.current_version, image_tag=effective_tag(db, t, core_tag),
|
||||
actor=actor, batch_id=batch_id, simulated=not (live_enabled() and kind in ("deploy", "offboard")))
|
||||
db.add(d)
|
||||
db.flush()
|
||||
dep_id = d.id
|
||||
|
||||
def work():
|
||||
ok = execute(dep_id, tenant_id, kind, http, oidc_client, core_tag, health)
|
||||
_finish(dep_id, "success" if ok else "failed")
|
||||
|
||||
th = threading.Thread(target=work, daemon=True)
|
||||
th.start()
|
||||
if wait:
|
||||
th.join()
|
||||
return dep_id
|
||||
|
||||
|
||||
def run_sequence(tenant_ids: list[str], kind: str, actor: str, *, batch: str, core_tag: str | None = None, http=None, oidc_client=None,
|
||||
health=None, canary: list[str] | None = None, wait: bool = False, on_done=None) -> list[int]:
|
||||
"""Tenants nacheinander (zuerst die Canary-Tenants), Abbruch beim ersten Fehler – der Rest wird „cancelled“.
|
||||
on_done(ok, results) wird nach dem Lauf aufgerufen (z. B. um ein Release abzuschließen)."""
|
||||
if kind in ("deploy", "offboard") and live_enabled(): # Live: nur freigegebene Tenants (siehe guard.py)
|
||||
skipped = [t for t in tenant_ids if not guard.live_allowed(t)]
|
||||
tenant_ids = [t for t in tenant_ids if guard.live_allowed(t)]
|
||||
if skipped:
|
||||
with session() as db:
|
||||
store.audit(db, actor, "deploy.skipped_not_allowed", None, ", ".join(skipped))
|
||||
canary = [c for c in (canary or []) if c in tenant_ids]
|
||||
order = canary + [t for t in tenant_ids if t not in canary]
|
||||
with session() as db:
|
||||
rows = []
|
||||
for tid in order:
|
||||
t = db.get(Tenant, tid)
|
||||
d = Deployment(tenant_id=tid, kind=kind, version=t.current_version, image_tag=effective_tag(db, t, core_tag), actor=actor,
|
||||
batch_id=batch, status="queued", simulated=not (live_enabled() and kind == "deploy"))
|
||||
db.add(d)
|
||||
rows.append(d)
|
||||
db.flush()
|
||||
dep_ids = [d.id for d in rows]
|
||||
|
||||
def seq():
|
||||
failed, results = False, {}
|
||||
for tid, did in zip(order, dep_ids):
|
||||
if failed:
|
||||
_append(did, "übersprungen – vorheriger Tenant fehlgeschlagen (Rollout abgebrochen)")
|
||||
_finish(did, "cancelled")
|
||||
results[tid] = "cancelled"
|
||||
continue
|
||||
with session() as db:
|
||||
db.get(Deployment, did).status = "running"
|
||||
ok = execute(did, tid, kind, http, oidc_client, core_tag, health)
|
||||
_finish(did, "success" if ok else "failed")
|
||||
results[tid] = "success" if ok else "failed"
|
||||
failed = not ok
|
||||
if tid in canary and ok and tid == canary[-1]:
|
||||
_append(did, "Canary erfolgreich – restliche Tenants folgen")
|
||||
if on_done:
|
||||
on_done(not failed, results)
|
||||
|
||||
th = threading.Thread(target=seq, daemon=True)
|
||||
th.start()
|
||||
if wait:
|
||||
th.join()
|
||||
return dep_ids
|
||||
|
||||
|
||||
def start_all(kind: str, actor: str, http=None, wait: bool = False, oidc_client=None, core_tag: str | None = None, health=None) -> tuple[str, list[int]]:
|
||||
"""Alle aktiven, nicht gepinnten Tenants nacheinander; beim ersten Fehler Abbruch."""
|
||||
batch = uuid.uuid4().hex[:10]
|
||||
with session() as db:
|
||||
ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status == "active").order_by(Tenant.id)) if not (core_tag and t.core_pin)]
|
||||
return batch, run_sequence(ids, kind, actor, batch=batch, core_tag=core_tag, http=http, oidc_client=oidc_client, health=health, wait=wait)
|
||||
35
app/services/guard.py
Normal file
35
app/services/guard.py
Normal file
@@ -0,0 +1,35 @@
|
||||
"""Schreib-Freigabe für externe Systeme (Mittwald, Login-API) – pro Tenant.
|
||||
|
||||
`DEPLOY_ALLOWED_TENANTS` (Env der Admin-Plattform) legt fest, welche Tenants live angefasst werden dürfen:
|
||||
(leer/nicht gesetzt) keiner – auch mit DEPLOY_LIVE=true wird nichts geschrieben
|
||||
demo-admin,demo2 nur diese
|
||||
* alle
|
||||
Greift bei Live-Deploy, Offboard, Rollout/Rollback und beim Schreiben im Login-System (OIDC-Abgleich/Löschen).
|
||||
Dry-Runs und Lesezugriffe sind nie betroffen. Damit lässt sich die Plattform schrittweise und gefahrlos parallel zum Altbetrieb einführen.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def allowed_tenants() -> set[str] | None:
|
||||
"""None = alle, sonst die erlaubten IDs (leere Menge = keiner)."""
|
||||
raw = os.getenv("DEPLOY_ALLOWED_TENANTS", "").strip()
|
||||
if raw == "*":
|
||||
return None
|
||||
return {x.strip() for x in raw.split(",") if x.strip()}
|
||||
|
||||
|
||||
def live_allowed(tenant_id: str) -> bool:
|
||||
a = allowed_tenants()
|
||||
return a is None or tenant_id in a
|
||||
|
||||
|
||||
def describe() -> str:
|
||||
a = allowed_tenants()
|
||||
return "alle Tenants" if a is None else (", ".join(sorted(a)) if a else "keiner")
|
||||
|
||||
|
||||
def denial(tenant_id: str) -> str:
|
||||
return (f"Tenant „{tenant_id}“ ist für Live-Schreibzugriffe nicht freigegeben (Env DEPLOY_ALLOWED_TENANTS = {describe()}). "
|
||||
"Dry-Runs sind weiterhin möglich.")
|
||||
151
app/services/importer.py
Normal file
151
app/services/importer.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""Importiert die Tenants der bestehenden transkript-platform (Git-Ordner) in die Datenbank.
|
||||
|
||||
Idempotent: ein Tenant, dessen Inhalt bereits identisch als aktuelle Version existiert,
|
||||
wird übersprungen. Secrets werden standardmäßig NUR als Schlüsselnamen importiert
|
||||
(Wert „extern/SOPS“); mit with_secrets=True werden sie via `sops -d` entschlüsselt
|
||||
und verschlüsselt in der DB abgelegt.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Deployment, Secret, Setting, Tenant, Version
|
||||
from app.services import crypto, store
|
||||
from app.services.samples import DEFAULT_SAMPLE, ROTARY_SAMPLE
|
||||
|
||||
IGNORE = {"secrets.enc.yaml", "tenant.json"}
|
||||
IGNORE_DIRS = {"__pycache__"}
|
||||
|
||||
# Bisherige Gitea-„Variables“ (docs/DEPLOYMENT.md) → geteilte Einstellungen
|
||||
DEFAULT_SETTINGS = {
|
||||
"base_domain": "onyscribe.de",
|
||||
"mittwald_project_id": "",
|
||||
"mittwald_stack_id": "",
|
||||
"registry": "",
|
||||
"image_name": "transkript-platform",
|
||||
"oidc_issuer": "https://login.onyva.dev",
|
||||
"onyva_login_api_base": "",
|
||||
"auth_enabled": "true",
|
||||
"app_cookie_secure": "true",
|
||||
"max_file_size_mb": "300",
|
||||
"cortecs_base_url": "https://api.cortecs.ai/v1",
|
||||
"llm_model": "claude-4-6-sonnet",
|
||||
"llm_max_tokens": "16000",
|
||||
"llm_temperature": "0.2",
|
||||
"smtp_host": "",
|
||||
"smtp_port": "587",
|
||||
"smtp_user": "",
|
||||
"smtp_from": "Gesprächsprotokoll",
|
||||
"mistral_transcribe_model": "voxtral-mini-latest",
|
||||
"mistral_diarize": "true",
|
||||
"mistral_timeout": "600",
|
||||
"core_image_tag": "latest",
|
||||
"core_supports_bundle": "false",
|
||||
"canary_tenants": "demo",
|
||||
"admin_public_url": "",
|
||||
"cost_api_base": "",
|
||||
"cost_tracking_enabled": "",
|
||||
"agent_model": "claude-4-6-sonnet",
|
||||
"agent_api": "messages",
|
||||
}
|
||||
|
||||
|
||||
def seed_settings(db: Session) -> None:
|
||||
for k, v in DEFAULT_SETTINGS.items():
|
||||
if db.get(Setting, k) is None:
|
||||
db.add(Setting(key=k, value=v))
|
||||
|
||||
|
||||
def _sops_keys(path: Path) -> list[str]:
|
||||
if not path.exists():
|
||||
return []
|
||||
data = yaml.safe_load(path.read_text()) or {}
|
||||
return [k for k in data if k != "sops"]
|
||||
|
||||
|
||||
def _sops_decrypt(path: Path) -> dict[str, str]:
|
||||
out = subprocess.run(["sops", "-d", "--output-type", "json", str(path)], capture_output=True, text=True, check=True)
|
||||
return {k: str(v) for k, v in json.loads(out.stdout).items()}
|
||||
|
||||
|
||||
def _upsert_secrets(db: Session, tenant_id: str | None, path: Path, with_secrets: bool) -> int:
|
||||
keys = _sops_keys(path)
|
||||
values = _sops_decrypt(path) if (with_secrets and keys) else {}
|
||||
n = 0
|
||||
for key in keys:
|
||||
row = db.scalar(select(Secret).where(Secret.tenant_id == tenant_id, Secret.key == key))
|
||||
if row is None:
|
||||
row = Secret(tenant_id=tenant_id, key=key)
|
||||
db.add(row)
|
||||
if key in values and values[key]:
|
||||
row.value_enc = crypto.encrypt(values[key])
|
||||
row.source = "manual"
|
||||
elif row.value_enc is None:
|
||||
row.source = "sops-import"
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def import_all(db: Session, root: Path, with_secrets: bool = False, force: bool = False) -> list[str]:
|
||||
log: list[str] = []
|
||||
seed_settings(db)
|
||||
tenants_dir = root / "tenants"
|
||||
src_rev = ""
|
||||
try:
|
||||
src_rev = subprocess.run(["git", "-C", str(root), "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
n = _upsert_secrets(db, None, root / "secrets" / "shared.enc.yaml", with_secrets)
|
||||
log.append(f"geteilte Secrets: {n} Schlüssel")
|
||||
|
||||
for tdir in sorted(p for p in tenants_dir.iterdir() if p.is_dir()):
|
||||
tid = tdir.name
|
||||
cfg_file = tdir / "tenant.json"
|
||||
if not cfg_file.exists():
|
||||
continue
|
||||
raw = json.loads(cfg_file.read_text(encoding="utf-8"))
|
||||
deploy = raw.pop("deploy", {}) or {}
|
||||
files: dict[str, bytes] = {}
|
||||
for p in sorted(tdir.rglob("*")):
|
||||
if p.is_dir() or p.name in IGNORE or any(part in IGNORE_DIRS for part in p.relative_to(tdir).parts):
|
||||
continue
|
||||
files[p.relative_to(tdir).as_posix()] = p.read_bytes()
|
||||
|
||||
t = db.get(Tenant, tid)
|
||||
if t is None:
|
||||
res = deploy.get("resources") or {}
|
||||
t = Tenant(
|
||||
id=tid, name=raw.get("name", tid), subdomain=deploy.get("subdomain", tid),
|
||||
cpu=float(res.get("cpu", 0.5)), memory=str(res.get("memory", "512M")),
|
||||
domain_status="verified", source="git", deployed_image="git-baked",
|
||||
preview_sample=ROTARY_SAMPLE if tid.startswith("rotary") else None,
|
||||
# Bestehende Clients wurden von Hand angelegt → nur „adoptieren“: fehlende Redirect-URI ergänzen, sonst nichts anfassen
|
||||
oidc={"mode": "adopt"},
|
||||
)
|
||||
db.add(t)
|
||||
db.flush()
|
||||
shas = {p: store.put_blob(db, data, p) for p, data in files.items()}
|
||||
cur = store.current_version(db, t)
|
||||
same = cur is not None and cur.files == shas and json.dumps(cur.config, sort_keys=True) == json.dumps(raw, sort_keys=True)
|
||||
if same and not force:
|
||||
log.append(f"{tid}: unverändert (v{t.current_version})")
|
||||
else:
|
||||
number = (t.current_version or 0) + 1
|
||||
db.add(Version(tenant_id=tid, number=number, message=f"Import aus transkript-platform @{src_rev}",
|
||||
author="import", config=raw, files=shas))
|
||||
t.current_version = number
|
||||
if t.deployed_version is None:
|
||||
t.deployed_version = number
|
||||
store.audit(db, "import", "tenant.import", tid, f"v{number} aus {src_rev}")
|
||||
log.append(f"{tid}: importiert als v{number} ({len(files)} Dateien)")
|
||||
k = _upsert_secrets(db, tid, tdir / "secrets.enc.yaml", with_secrets)
|
||||
log.append(f" Secrets: {k} Schlüssel")
|
||||
db.flush()
|
||||
return log
|
||||
121
app/services/llm.py
Normal file
121
app/services/llm.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""Zugang zu Cortecs (OpenAI-kompatibel + Anthropic-Messages) für Agent und Prompt-Test.
|
||||
|
||||
Der Key kommt aus dem geteilten Secret CORTECS_API_KEY (verschlüsselt in der DB) oder der Env CORTECS_API_KEY.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Secret, Setting
|
||||
from app.services import crypto
|
||||
|
||||
DEFAULT_BASE = "https://api.cortecs.ai/v1"
|
||||
|
||||
|
||||
class LLMNotConfigured(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _setting(db: Session, key: str, default: str = "") -> str:
|
||||
s = db.get(Setting, key)
|
||||
return s.value if s and s.value else default
|
||||
|
||||
|
||||
def key_info(db: Session, tenant_id: str | None = None) -> tuple[str, str, str | None]:
|
||||
"""(key, quelle, fehler). Reihenfolge: geteiltes Secret (DB) → Env CORTECS_API_KEY → Key des Tenants (Rückfall).
|
||||
Quelle: 'shared' | 'env' | 'tenant' | 'none'. fehler: z. B. nicht entschlüsselbar (SECRET_KEY geändert?)."""
|
||||
error = None
|
||||
rows = [("shared", db.scalar(select(Secret).where(Secret.tenant_id.is_(None), Secret.key == "CORTECS_API_KEY")))]
|
||||
for source, row in rows:
|
||||
if row and row.value_enc:
|
||||
try:
|
||||
return crypto.decrypt(row.value_enc), source, None
|
||||
except Exception: # noqa: BLE001 – falscher SECRET_KEY o. Ä.
|
||||
error = "Geteilter CORTECS_API_KEY in der DB ist nicht entschlüsselbar (SECRET_KEY geändert?) – bitte neu setzen."
|
||||
env = os.getenv("CORTECS_API_KEY", "").strip()
|
||||
if env:
|
||||
return env, "env", error
|
||||
if tenant_id:
|
||||
row = db.scalar(select(Secret).where(Secret.tenant_id == tenant_id, Secret.key == "CORTECS_API_KEY"))
|
||||
if row and row.value_enc:
|
||||
try:
|
||||
return crypto.decrypt(row.value_enc), "tenant", error
|
||||
except Exception: # noqa: BLE001
|
||||
error = error or "CORTECS_API_KEY dieses Tenants ist nicht entschlüsselbar."
|
||||
return "", "none", error
|
||||
|
||||
|
||||
def api_key(db: Session, tenant_id: str | None = None) -> str:
|
||||
return key_info(db, tenant_id)[0]
|
||||
|
||||
|
||||
def base_url(db: Session) -> str:
|
||||
return _setting(db, "cortecs_base_url", DEFAULT_BASE).rstrip("/")
|
||||
|
||||
|
||||
def configured(db: Session, tenant_id: str | None = None) -> bool:
|
||||
return bool(api_key(db, tenant_id))
|
||||
|
||||
|
||||
def require_key(db: Session, tenant_id: str | None = None) -> str:
|
||||
k, _, err = key_info(db, tenant_id)
|
||||
if not k:
|
||||
raise LLMNotConfigured(err or "CORTECS_API_KEY fehlt – unter Einstellungen → Geteilte Secrets setzen (oder Env CORTECS_API_KEY).")
|
||||
return k
|
||||
|
||||
|
||||
def list_models(db: Session) -> list[dict]:
|
||||
"""GET /v1/models – Rohdaten von Cortecs, auf die für die Auswahl nötigen Felder reduziert."""
|
||||
key = require_key(db)
|
||||
r = httpx.get(f"{base_url(db)}/models", headers={"Authorization": f"Bearer {key}"}, timeout=30)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
items = data.get("data") if isinstance(data, dict) else data
|
||||
out = []
|
||||
for m in items or []:
|
||||
if isinstance(m, str):
|
||||
m = {"id": m}
|
||||
caps = {k: v for k, v in m.items() if isinstance(v, (bool, list, dict)) or k in ("context_size", "context_length")}
|
||||
out.append({"id": m.get("id") or m.get("name"), "raw": caps,
|
||||
"tools": _supports_tools(m)})
|
||||
return sorted((o for o in out if o["id"]), key=lambda o: o["id"])
|
||||
|
||||
|
||||
def _supports_tools(m: dict) -> bool | None:
|
||||
"""Cortecs dokumentiert die Feldnamen nicht eindeutig → mehrere plausible Varianten prüfen; None = unbekannt."""
|
||||
flat = {}
|
||||
def walk(prefix, v):
|
||||
if isinstance(v, dict):
|
||||
for k, x in v.items():
|
||||
walk(f"{prefix}.{k}".lower(), x)
|
||||
elif isinstance(v, list):
|
||||
flat[prefix] = [str(i).lower() for i in v]
|
||||
else:
|
||||
flat[prefix] = v
|
||||
walk("", m)
|
||||
for k, v in flat.items():
|
||||
if "tool" in k or "function" in k:
|
||||
return bool(v)
|
||||
if isinstance(v, list) and any("tool" in i or "function" in i for i in v):
|
||||
return True
|
||||
return None
|
||||
|
||||
|
||||
def summarize(db: Session, system_prompt: str, user_content: str, model: str | None = None, tenant_id: str | None = None) -> dict:
|
||||
"""Ein Protokoll-Lauf wie im Core (Chat Completions), für den Prompt-Test."""
|
||||
from openai import OpenAI
|
||||
client = OpenAI(api_key=require_key(db, tenant_id), base_url=base_url(db), max_retries=2, timeout=300)
|
||||
model = model or _setting(db, "llm_model", "claude-4-6-sonnet")
|
||||
resp = client.chat.completions.create(
|
||||
model=model, temperature=float(_setting(db, "llm_temperature", "0.2")),
|
||||
max_tokens=int(_setting(db, "llm_max_tokens", "16000")), stream=False,
|
||||
messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_content}],
|
||||
)
|
||||
text = "\n".join(c.message.content for c in resp.choices if c.message and c.message.content).strip()
|
||||
u = getattr(resp, "usage", None)
|
||||
return {"text": text, "model": model, "input_tokens": getattr(u, "prompt_tokens", 0), "output_tokens": getattr(u, "completion_tokens", 0),
|
||||
"truncated": any(getattr(c, "finish_reason", "") in ("length", "max_tokens") for c in resp.choices)}
|
||||
159
app/services/mittwald.py
Normal file
159
app/services/mittwald.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""Mittwald-API-Client (Port der Logik aus transkript-platform/infra/deploy.py).
|
||||
|
||||
Sicherheitsnetze wie dort: differenzielles PATCH (nie PUT), Namensraum `app-`, Post-Write-Guard.
|
||||
Zusätzlich: `live=False` (Standard) blockiert JEDEN schreibenden Aufruf – es wird nur geloggt, was passieren würde.
|
||||
Lesende Aufrufe laufen nur, wenn ein Token gesetzt ist.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Callable
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class DeployError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class DryRunSkip(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def mask_env(env: dict) -> dict:
|
||||
import re
|
||||
return {k: ("••••" if re.search(r"(SECRET|PASSWORD|KEY|TOKEN)$", k) else v) for k, v in env.items()}
|
||||
|
||||
|
||||
class Mittwald:
|
||||
def __init__(self, token: str, base: str, live: bool, log: Callable[[str], None], http: Callable | None = None):
|
||||
self.token, self.base, self.live, self.log = token, base.rstrip("/"), live, log
|
||||
self._http = http or self._real_http
|
||||
self._project_uuid: str | None = None
|
||||
|
||||
@staticmethod
|
||||
def _real_http(method, url, headers, body):
|
||||
r = httpx.request(method, url, headers=headers, json=body, timeout=60)
|
||||
try:
|
||||
data = r.json() if r.content else None
|
||||
except ValueError:
|
||||
data = None
|
||||
return r.status_code, data, r.text[:500]
|
||||
|
||||
def api(self, method: str, path: str, body: dict | None = None, *, write: bool, fatal: bool = True):
|
||||
if write and not self.live:
|
||||
self.log(f"[DRY-RUN] {method} {path}")
|
||||
if body is not None:
|
||||
shown = json.loads(json.dumps(body))
|
||||
for svc in (shown.get("services") or {}).values():
|
||||
if isinstance(svc, dict) and "environment" in svc:
|
||||
svc["environment"] = mask_env(svc["environment"])
|
||||
self.log(" body: " + json.dumps(shown, ensure_ascii=False)[:1500])
|
||||
raise DryRunSkip()
|
||||
if not self.token:
|
||||
if not self.live:
|
||||
self.log(f"(übersprungen: lesender Aufruf {method} {path} – kein MITTWALD_API_TOKEN)")
|
||||
raise DryRunSkip()
|
||||
raise DeployError("MITTWALD_API_TOKEN nicht gesetzt.")
|
||||
status, data, text = self._http(method, f"{self.base}{path}", {"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"}, body)
|
||||
if status >= 400:
|
||||
msg = f"{method} {path} → HTTP {status}: {text}"
|
||||
if fatal:
|
||||
raise DeployError(msg)
|
||||
self.log(f"WARNUNG (nicht fatal): {msg}")
|
||||
return None
|
||||
return data
|
||||
|
||||
def try_api(self, *a, **kw):
|
||||
try:
|
||||
return self.api(*a, **kw)
|
||||
except DryRunSkip:
|
||||
return None
|
||||
|
||||
# ── Stack ──
|
||||
def resolve_stack_id(self, project_id: str, stack_id: str) -> str:
|
||||
if stack_id:
|
||||
return stack_id
|
||||
if not project_id:
|
||||
if not self.live:
|
||||
self.log("(Hinweis: Mittwald PROJECT_ID/STACK_ID sind noch nicht in den Einstellungen gesetzt – Platzhalter im Dry-Run)")
|
||||
return "<STACK_ID>"
|
||||
raise DeployError("Mittwald PROJECT_ID (oder STACK_ID) fehlt in den Einstellungen.")
|
||||
try:
|
||||
stacks = self.api("GET", f"/projects/{project_id}/stacks/", write=False)
|
||||
except DryRunSkip:
|
||||
return "<STACK_ID>"
|
||||
items = stacks if isinstance(stacks, list) else (stacks or {}).get("items", [])
|
||||
for s in items:
|
||||
if str(s.get("description") or s.get("name") or "").lower() == "default":
|
||||
return s["id"]
|
||||
if items:
|
||||
return items[0]["id"]
|
||||
raise DeployError("Kein Stack im Projekt gefunden.")
|
||||
|
||||
def get_stack(self, stack_id: str) -> dict | None:
|
||||
return self.try_api("GET", f"/stacks/{stack_id}/", write=False)
|
||||
|
||||
@staticmethod
|
||||
def services_of(stack: dict | None) -> dict[str, dict]:
|
||||
if not stack:
|
||||
return {}
|
||||
svcs = stack.get("services")
|
||||
if isinstance(svcs, dict):
|
||||
return svcs
|
||||
if isinstance(svcs, list):
|
||||
return {(s.get("serviceName") or s.get("name") or s.get("id")): s for s in svcs if (s.get("serviceName") or s.get("name") or s.get("id"))}
|
||||
return {}
|
||||
|
||||
def project_uuid(self, project_id: str) -> str:
|
||||
if self._project_uuid:
|
||||
return self._project_uuid
|
||||
proj = self.try_api("GET", f"/projects/{project_id}/", write=False, fatal=False)
|
||||
self._project_uuid = (proj or {}).get("id") or project_id
|
||||
return self._project_uuid
|
||||
|
||||
# ── Operationen ──
|
||||
def deploy_service(self, *, stack_id: str, service: str, volume: str, definition: dict, container_port: str,
|
||||
project_id: str, hostname: str | None) -> None:
|
||||
stack = self.get_stack(stack_id)
|
||||
existing = set(self.services_of(stack))
|
||||
protected = existing - {service}
|
||||
if protected:
|
||||
self.log(f"geschützte bestehende Services (bleiben unangetastet): {sorted(protected)}")
|
||||
self.try_api("PATCH", f"/stacks/{stack_id}/", {"volumes": {volume: {"name": volume}}}, write=True)
|
||||
self.try_api("PATCH", f"/stacks/{stack_id}/", {"services": {service: definition}}, write=True)
|
||||
svc_id = "<service-id>"
|
||||
if self.live:
|
||||
after = self.services_of(self.get_stack(stack_id))
|
||||
missing = protected - set(after)
|
||||
if missing:
|
||||
raise DeployError(f"ABBRUCH: bestehende Services verschwunden nach PATCH: {sorted(missing)}")
|
||||
svc_id = after.get(service, {}).get("id", service)
|
||||
self.try_api("POST", f"/stacks/{stack_id}/services/{svc_id}/actions/recreate/", write=True)
|
||||
self.log("Post-Write-Guard: alle vorherigen Services noch vorhanden ✔")
|
||||
if hostname:
|
||||
self.ensure_ingress(project_id, hostname, svc_id, container_port)
|
||||
|
||||
def ensure_ingress(self, project_id: str, hostname: str, container_id: str, port: str) -> None:
|
||||
listing = self.try_api("GET", f"/projects/{project_id}/ingresses/", write=False, fatal=False)
|
||||
items = listing if isinstance(listing, list) else (listing or {}).get("items", [])
|
||||
if any(i.get("hostname") == hostname for i in items):
|
||||
self.log(f"Ingress {hostname} existiert bereits → ok")
|
||||
return
|
||||
body = {"hostname": hostname, "projectId": self.project_uuid(project_id),
|
||||
"paths": [{"path": "/", "target": {"container": {"id": container_id, "portProtocol": f"{port}/tcp"}}}]}
|
||||
res = self.try_api("POST", "/ingresses", body, write=True, fatal=False)
|
||||
if self.live:
|
||||
self.log(f"Ingress {hostname} angelegt (Auto-TLS)" if res is not None else
|
||||
f"⚠ Ingress {hostname} NICHT angelegt – Token braucht Domain/Ingress-Recht (Service läuft trotzdem).")
|
||||
|
||||
def offboard_service(self, *, stack_id: str, service: str, volume: str, project_id: str, hostname: str | None) -> None:
|
||||
self.try_api("PATCH", f"/stacks/{stack_id}/", {"services": {service: {}}}, write=True)
|
||||
self.try_api("PATCH", f"/stacks/{stack_id}/", {"volumes": {volume: {}}}, write=True)
|
||||
if hostname:
|
||||
listing = self.try_api("GET", f"/projects/{project_id}/ingresses/", write=False, fatal=False)
|
||||
items = listing if isinstance(listing, list) else (listing or {}).get("items", [])
|
||||
for i in items:
|
||||
if i.get("hostname") == hostname:
|
||||
self.try_api("DELETE", f"/ingresses/{i['id']}", write=True, fatal=False)
|
||||
self.log(f"Ingress {hostname} entfernt")
|
||||
312
app/services/onyva_login.py
Normal file
312
app/services/onyva_login.py
Normal file
@@ -0,0 +1,312 @@
|
||||
"""Abgleich der OIDC-Clients mit der Developer-API von login.onyva.dev (`/api/v1`, siehe LoginSeite/docs/developer-api.md).
|
||||
|
||||
Pro Tenant gibt es einen Soll-Zustand (`tenant.oidc`). Bei jedem Deploy wird der Ist-Zustand gelesen und bei Bedarf angelegt/angepasst:
|
||||
|
||||
mode = off nichts anfassen
|
||||
mode = adopt (Standard für importierte Altclients) nur sicherstellen, dass die Redirect-URIs des Tenants enthalten sind
|
||||
mode = managed Redirect-URIs, auth_mode, totp_mode, allow_registration, backchannel_logout_uri exakt wie konfiguriert
|
||||
|
||||
Freigaben (Memberships): `members` werden angelegt/aktualisiert; mit members_mode = exact werden zusätzliche Freigaben entfernt.
|
||||
|
||||
Sicherheit: Lesen (GET) läuft immer, wenn ein Key da ist. SCHREIBEN nur mit apply=True – und apply erlaubt die Plattform nur,
|
||||
wenn OIDC_SYNC_LIVE=true oder DEPLOY_LIVE=true. Das `client_secret` ist nur bei Anlage/Rotation sichtbar und wird SOFORT verschlüsselt gespeichert.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import session
|
||||
from app.models import Secret, Setting, Tenant
|
||||
from app.services import crypto, guard, store
|
||||
|
||||
MODES = ("off", "adopt", "managed")
|
||||
AUTH_MODES = ("required", "optional", "disabled")
|
||||
TOTP_MODES = ("disabled", "optional", "required")
|
||||
ROLES = ("viewer", "editor", "admin")
|
||||
DEFAULTS = {"mode": "adopt", "auth_mode": "required", "totp_mode": "disabled", "allow_registration": False,
|
||||
"backchannel_logout_uri": None, "members": [], "members_mode": "add", "last_sync": None}
|
||||
|
||||
|
||||
class OnyvaError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def key_present() -> bool:
|
||||
return bool(os.getenv("ONYVA_LOGIN_API_KEY", "").strip())
|
||||
|
||||
|
||||
def writes_allowed() -> bool:
|
||||
return os.getenv("OIDC_SYNC_LIVE", "false").lower() == "true" or os.getenv("DEPLOY_LIVE", "false").lower() == "true"
|
||||
|
||||
|
||||
def base_url(db) -> str:
|
||||
s = db.get(Setting, "onyva_login_api_base")
|
||||
if s and s.value:
|
||||
return s.value.rstrip("/")
|
||||
issuer = db.get(Setting, "oidc_issuer")
|
||||
return f"{(issuer.value if issuer and issuer.value else 'https://login.onyva.dev').rstrip('/')}/api/v1"
|
||||
|
||||
|
||||
class OnyvaLogin:
|
||||
"""Dünner Client für /api/v1/apps. `http(method, url, headers, body) -> (status, data, headers)` ist in Tests ersetzbar."""
|
||||
|
||||
def __init__(self, base: str, key: str, http: Callable | None = None, sleep: Callable[[float], None] = time.sleep):
|
||||
self.base, self.key, self._http, self._sleep = base.rstrip("/"), key, http or self._real, sleep
|
||||
|
||||
@staticmethod
|
||||
def _real(method, url, headers, body):
|
||||
r = httpx.request(method, url, headers=headers, json=body, timeout=30)
|
||||
try:
|
||||
data = r.json() if r.content else None
|
||||
except ValueError:
|
||||
data = None
|
||||
return r.status_code, data, dict(r.headers)
|
||||
|
||||
def _req(self, method: str, path: str, body: dict | None = None):
|
||||
for attempt in range(4):
|
||||
status, data, headers = self._http(method, f"{self.base}{path}", {"Authorization": f"Bearer {self.key}", "Content-Type": "application/json"}, body)
|
||||
if status == 429 and attempt < 3: # Rate-Limit: 120 Requests / 60 s pro Key
|
||||
self._sleep(min(int({k.lower(): v for k, v in headers.items()}.get("retry-after", 5)), 60))
|
||||
continue
|
||||
if status == 401:
|
||||
raise OnyvaError("API-Key ungültig, widerrufen oder abgelaufen (401).")
|
||||
return status, data
|
||||
raise OnyvaError("Rate-Limit des Login-Systems (429) dauerhaft überschritten.")
|
||||
|
||||
def _ok(self, method, path, body=None, ok=(200, 201, 204)):
|
||||
status, data = self._req(method, path, body)
|
||||
if status not in ok:
|
||||
detail = data.get("detail") if isinstance(data, dict) else data
|
||||
raise OnyvaError(f"{method} {path} → HTTP {status}: {str(detail)[:300]}")
|
||||
return data
|
||||
|
||||
def list_apps(self) -> list[dict]:
|
||||
return (self._ok("GET", "/apps") or {}).get("apps", [])
|
||||
|
||||
def get_app(self, cid: str) -> dict | None:
|
||||
status, data = self._req("GET", f"/apps/{cid}")
|
||||
if status == 404:
|
||||
return None
|
||||
if status != 200:
|
||||
raise OnyvaError(f"GET /apps/{cid} → HTTP {status}")
|
||||
return data
|
||||
|
||||
def create_app(self, body: dict) -> dict:
|
||||
return self._ok("POST", "/apps", body, ok=(201, 200))
|
||||
|
||||
def patch_app(self, cid: str, body: dict) -> dict:
|
||||
return self._ok("PATCH", f"/apps/{cid}", body)
|
||||
|
||||
def delete_app(self, cid: str) -> None:
|
||||
self._ok("DELETE", f"/apps/{cid}", ok=(204, 200))
|
||||
|
||||
def rotate_secret(self, cid: str) -> dict:
|
||||
return self._ok("POST", f"/apps/{cid}/rotate-secret")
|
||||
|
||||
def memberships(self, cid: str) -> list[dict]:
|
||||
return (self._ok("GET", f"/apps/{cid}/memberships") or {}).get("memberships", [])
|
||||
|
||||
def put_membership(self, cid: str, email: str, role: str) -> bool:
|
||||
status, _ = self._req("PUT", f"/apps/{cid}/memberships", {"user_email": email, "role": role})
|
||||
if status == 404:
|
||||
return False # Nutzer existiert im Login nicht
|
||||
if status not in (200, 201):
|
||||
raise OnyvaError(f"PUT memberships → HTTP {status}")
|
||||
return True
|
||||
|
||||
def delete_membership(self, cid: str, email: str) -> None:
|
||||
self._req("DELETE", f"/apps/{cid}/memberships/{email}")
|
||||
|
||||
|
||||
def client_from_env(db, http=None) -> OnyvaLogin | None:
|
||||
key = os.getenv("ONYVA_LOGIN_API_KEY", "").strip()
|
||||
return OnyvaLogin(base_url(db), key, http) if key else None
|
||||
|
||||
|
||||
# ── Soll-Zustand ────────────────────────────────────────────────────────────
|
||||
|
||||
def config_of(t: Tenant) -> dict:
|
||||
return {**DEFAULTS, **(t.oidc or {})}
|
||||
|
||||
|
||||
def validate_config(cfg: dict) -> dict:
|
||||
out = {**DEFAULTS, **cfg}
|
||||
if out["mode"] not in MODES:
|
||||
raise ValueError("mode: off | adopt | managed")
|
||||
if out["auth_mode"] not in AUTH_MODES:
|
||||
raise ValueError("auth_mode: required | optional | disabled")
|
||||
if out["totp_mode"] not in TOTP_MODES:
|
||||
raise ValueError("totp_mode: disabled | optional | required")
|
||||
if out["members_mode"] not in ("add", "exact"):
|
||||
raise ValueError("members_mode: add | exact")
|
||||
out["allow_registration"] = bool(out["allow_registration"])
|
||||
out["backchannel_logout_uri"] = (out["backchannel_logout_uri"] or "").strip() or None
|
||||
if out["backchannel_logout_uri"] and not out["backchannel_logout_uri"].startswith("https://"):
|
||||
raise ValueError("Backchannel-Logout-URI muss https:// sein und öffentlich erreichbar")
|
||||
members, seen = [], set()
|
||||
for m in out["members"] or []:
|
||||
email, role = str(m.get("email", "")).strip().lower(), m.get("role", "viewer")
|
||||
if not re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", email):
|
||||
raise ValueError(f"Ungültige E-Mail: {email!r}")
|
||||
if role not in ROLES:
|
||||
raise ValueError(f"Rolle von {email}: viewer | editor | admin")
|
||||
if email not in seen:
|
||||
seen.add(email)
|
||||
members.append({"email": email, "role": role})
|
||||
out["members"] = members
|
||||
out.pop("last_sync", None)
|
||||
return out
|
||||
|
||||
|
||||
def desired_redirects(base_domain: str, t: Tenant) -> list[str]:
|
||||
hosts = [f"{t.subdomain or t.id}.{base_domain}"]
|
||||
if t.custom_domain:
|
||||
hosts.append(t.custom_domain)
|
||||
return [f"https://{h}/auth/callback" for h in hosts]
|
||||
|
||||
|
||||
def diff_app(actual: dict, want: dict, mode: str) -> dict:
|
||||
"""Welche PATCH-Felder wären nötig? adopt: nur fehlende URIs ergänzen; managed: alles exakt."""
|
||||
changes: dict = {}
|
||||
have, need = list(actual.get("redirect_uris") or []), want["redirect_uris"]
|
||||
if mode == "adopt":
|
||||
missing = [u for u in need if u not in have]
|
||||
if missing:
|
||||
changes["redirect_uris"] = have + missing
|
||||
elif mode == "managed":
|
||||
if set(have) != set(need):
|
||||
changes["redirect_uris"] = need
|
||||
for f in ("auth_mode", "totp_mode", "allow_registration", "backchannel_logout_uri"):
|
||||
if actual.get(f) != want[f]:
|
||||
changes[f] = want[f]
|
||||
return changes
|
||||
|
||||
|
||||
def _secret(db, tenant_id: str, key: str) -> str | None:
|
||||
row = db.scalar(select(Secret).where(Secret.tenant_id == tenant_id, Secret.key == key))
|
||||
return crypto.decrypt(row.value_enc) if row and row.value_enc else None
|
||||
|
||||
|
||||
def _store_credentials(tenant_id: str, client_id: str | None, client_secret: str | None, actor="onyva-login-sync") -> None:
|
||||
"""Sofort und in eigener Transaktion sichern – das Secret ist danach nicht mehr abrufbar."""
|
||||
from app.services import tenants as tsvc
|
||||
with session() as db:
|
||||
if client_id:
|
||||
tsvc.set_secret(db, tenant_id, "OIDC_CLIENT_ID", client_id, actor)
|
||||
if client_secret:
|
||||
tsvc.set_secret(db, tenant_id, "OIDC_CLIENT_SECRET", client_secret, actor)
|
||||
|
||||
|
||||
# ── Abgleich ────────────────────────────────────────────────────────────────
|
||||
|
||||
def sync_tenant(tenant_id: str, *, apply: bool, log: Callable[[str], None] = lambda m: None, client: OnyvaLogin | None = None,
|
||||
rotate: bool = False, actor: str = "system") -> dict:
|
||||
"""Gleicht den OIDC-Client des Tenants ab. apply=False → nur lesen und Plan melden.
|
||||
Rückgabe: {ok, applied, skipped, client_id, actions:[{kind,detail}], warnings:[...], created, error}."""
|
||||
rep = {"ok": True, "applied": False, "skipped": None, "client_id": None, "actions": [], "warnings": [], "created": False, "error": None}
|
||||
act = lambda kind, detail: (rep["actions"].append({"kind": kind, "detail": detail}), log(f"OIDC: {detail}")) # noqa: E731
|
||||
warn = lambda msg: (rep["warnings"].append(msg), log(f"OIDC ⚠ {msg}")) # noqa: E731
|
||||
|
||||
if apply and not writes_allowed():
|
||||
rep.update(ok=False, error="Schreibzugriffe auf das Login-System sind gesperrt (OIDC_SYNC_LIVE/DEPLOY_LIVE nicht aktiv).")
|
||||
return rep
|
||||
if apply and not guard.live_allowed(tenant_id):
|
||||
rep.update(ok=False, error=guard.denial(tenant_id))
|
||||
return rep
|
||||
try:
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tenant_id)
|
||||
cfg = config_of(t)
|
||||
base_domain = (db.get(Setting, "base_domain").value if db.get(Setting, "base_domain") else "example.de")
|
||||
want = {"name": f"Onyscribe – {t.name}", "redirect_uris": desired_redirects(base_domain, t), "auth_mode": cfg["auth_mode"],
|
||||
"totp_mode": cfg["totp_mode"], "allow_registration": cfg["allow_registration"], "backchannel_logout_uri": cfg["backchannel_logout_uri"]}
|
||||
cid, has_secret = _secret(db, tenant_id, "OIDC_CLIENT_ID"), bool(_secret(db, tenant_id, "OIDC_CLIENT_SECRET"))
|
||||
client = client or client_from_env(db)
|
||||
mode = cfg["mode"]
|
||||
if mode == "off":
|
||||
rep["skipped"] = "OIDC-Verwaltung ist für diesen Tenant aus."
|
||||
log("OIDC: Verwaltung aus – nichts zu tun")
|
||||
return rep
|
||||
if client is None:
|
||||
rep["skipped"] = "ONYVA_LOGIN_API_KEY nicht gesetzt."
|
||||
rep["ok"] = not apply # beim echten Deploy ist ein fehlender Key ein Fehler, im Dry-Run nur ein Hinweis
|
||||
rep["error"] = None if rep["ok"] else rep["skipped"]
|
||||
log("OIDC: (übersprungen – ONYVA_LOGIN_API_KEY nicht gesetzt)")
|
||||
return rep
|
||||
|
||||
app = client.get_app(cid) if cid else None
|
||||
new_app = app is None
|
||||
verb = "" if apply else "würde "
|
||||
if app is None:
|
||||
if cid:
|
||||
warn(f"Client {cid} existiert im Login nicht mehr – es wird ein neuer angelegt.")
|
||||
body = {k: want[k] for k in ("name", "redirect_uris", "auth_mode", "totp_mode", "allow_registration", "backchannel_logout_uri")}
|
||||
act("create", f"{verb}App „{want['name']}“ anlegen (Redirect: {', '.join(want['redirect_uris'])}; auth_mode={want['auth_mode']})")
|
||||
if apply:
|
||||
created = client.create_app(body)
|
||||
cid = created["client_id"]
|
||||
_store_credentials(tenant_id, cid, created.get("client_secret"), actor) # ZUERST sichern
|
||||
rep["created"], has_secret = True, True
|
||||
log(f"OIDC: App angelegt → {cid} (Secret verschlüsselt gespeichert)")
|
||||
app = {"redirect_uris": want["redirect_uris"], "auth_mode": want["auth_mode"]}
|
||||
else:
|
||||
changes = diff_app(app, want, mode)
|
||||
if changes:
|
||||
act("patch", f"{verb}App {cid} anpassen: " + ", ".join(f"{k}={v}" for k, v in changes.items()))
|
||||
if apply:
|
||||
client.patch_app(cid, changes)
|
||||
else:
|
||||
log(f"OIDC: App {cid} ist abgeglichen")
|
||||
rep["client_id"] = cid
|
||||
|
||||
if not has_secret:
|
||||
warn("Client-Secret ist nicht in der Admin-DB (beim Login nicht abrufbar) – eintragen oder „Secret rotieren“.")
|
||||
if rotate and cid:
|
||||
act("rotate", f"{verb}Secret von {cid} rotieren (altes sofort ungültig)")
|
||||
if apply:
|
||||
_store_credentials(tenant_id, None, client.rotate_secret(cid)["client_secret"], actor)
|
||||
|
||||
# Freigaben
|
||||
members = cfg["members"]
|
||||
effective_auth = want["auth_mode"] if mode == "managed" else (app.get("auth_mode") or want["auth_mode"])
|
||||
if (members or cfg["members_mode"] == "exact") and (cid or new_app):
|
||||
current = {} if new_app else {m["email"].lower(): m["role"] for m in client.memberships(cid)}
|
||||
for m in members:
|
||||
if current.get(m["email"]) != m["role"]:
|
||||
act("member_put", f"{verb}{m['email']} als {m['role']} freigeben")
|
||||
if apply and not client.put_membership(cid, m["email"], m["role"]):
|
||||
warn(f"{m['email']} existiert im Login nicht – Nutzer dort anlegen, dann erneut abgleichen.")
|
||||
if cfg["members_mode"] == "exact":
|
||||
for email in sorted(set(current) - {m["email"] for m in members}):
|
||||
act("member_delete", f"{verb}Freigabe von {email} entziehen")
|
||||
if apply:
|
||||
client.delete_membership(cid, email)
|
||||
elif effective_auth == "required" and not members:
|
||||
log("OIDC: auth_mode=required ohne gepflegte Freigaben – bestehende Freigaben bleiben unverändert")
|
||||
rep["applied"] = apply and bool(rep["actions"])
|
||||
except OnyvaError as exc:
|
||||
rep.update(ok=False, error=str(exc))
|
||||
log(f"OIDC ✘ {exc}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
rep.update(ok=False, error=f"{type(exc).__name__}: {exc}")
|
||||
log(f"OIDC ✘ {rep['error']}")
|
||||
if apply:
|
||||
_record(tenant_id, rep)
|
||||
return rep
|
||||
|
||||
|
||||
def _record(tenant_id: str, rep: dict) -> None:
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tenant_id)
|
||||
cfg = dict(t.oidc or {})
|
||||
cfg["last_sync"] = {"ts": datetime.now(timezone.utc).isoformat(), "ok": rep["ok"], "created": rep["created"], "error": rep["error"],
|
||||
"actions": [a["detail"] for a in rep["actions"]], "warnings": rep["warnings"]}
|
||||
t.oidc = cfg
|
||||
store.audit(db, "system", "oidc.sync", tenant_id, "ok" if rep["ok"] else f"Fehler: {rep['error']}")
|
||||
114
app/services/preflight.py
Normal file
114
app/services/preflight.py
Normal file
@@ -0,0 +1,114 @@
|
||||
"""Vorflug-Check: ist ein Tenant bereit für einen Live-Deploy? Liest nur (Mittwald-GET, Login-API-GET, eigener Health) und schreibt nichts.
|
||||
|
||||
Jeder Punkt: {id, level: ok|warn|fail|info, label, detail, fix}. Ein Deploy ist „bereit“, wenn es kein `fail` gibt.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import session
|
||||
from app.models import Secret, Tenant
|
||||
from app.services import contract, deploy, guard, onyva_login, preview, store
|
||||
from app.services.mittwald import DeployError, DryRunSkip, Mittwald
|
||||
|
||||
|
||||
def _item(id_, level, label, detail="", fix=""):
|
||||
return {"id": id_, "level": level, "label": label, "detail": detail, "fix": fix}
|
||||
|
||||
|
||||
def run(tenant_id: str, *, mw_http=None, oidc_client=None, self_get=None) -> dict:
|
||||
out: list[dict] = []
|
||||
add = lambda *a, **k: out.append(_item(*a, **k)) # noqa: E731
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tenant_id)
|
||||
cfg, _ = store.content(db, t, "current")
|
||||
draft, source, pin = bool(t.draft), t.source, t.core_pin
|
||||
base_domain, project, stack = deploy.setting(db, "base_domain"), deploy.setting(db, "mittwald_project_id"), deploy.setting(db, "mittwald_stack_id")
|
||||
registry, admin_url = deploy.setting(db, "registry"), deploy.setting(db, "admin_public_url")
|
||||
bundle_ok = deploy.setting(db, "core_supports_bundle", "false") == "true"
|
||||
tag = deploy.effective_tag(db, t)
|
||||
host = deploy.host_for(db, t)
|
||||
missing = deploy.missing_secrets(db, t)
|
||||
oc = onyva_login.config_of(t)
|
||||
has_oidc_secrets = all(db.scalar(select(Secret).where(Secret.tenant_id == tenant_id, Secret.key == k, Secret.value_enc.is_not(None))) is not None
|
||||
for k in ("OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET"))
|
||||
stt_provider = (cfg.get("stt") or {}).get("provider") or "mistral"
|
||||
|
||||
# Freigaben / Modus
|
||||
add("live", "ok" if deploy.live_enabled() else "warn", "DEPLOY_LIVE", "aktiv" if deploy.live_enabled() else "nicht aktiv – nur Dry-Run möglich",
|
||||
"" if deploy.live_enabled() else "Env DEPLOY_LIVE=true der Admin-Plattform setzen und neu starten (erst nach erfolgreichem Dry-Run).")
|
||||
add("allow", "ok" if guard.live_allowed(tenant_id) else "warn", "Live-Freigabe für diesen Tenant", f"DEPLOY_ALLOWED_TENANTS = {guard.describe()}",
|
||||
"" if guard.live_allowed(tenant_id) else f"Env DEPLOY_ALLOWED_TENANTS um „{tenant_id}“ ergänzen.")
|
||||
# Vertrag / Core
|
||||
add("contract", "ok" if contract.supported() else "fail", "Core-Vertrag", f"Admin-Core {contract.sha()[:10]}, Vertrag v{contract.contract_version()}",
|
||||
"" if contract.supported() else "Admin-Plattform aktualisieren.")
|
||||
hex_tag = deploy._hex_tag(tag)
|
||||
add("tag", "ok" if hex_tag else "warn", "Core-Image-Tag", f":{tag}" + (" (gepinnt)" if pin else ""),
|
||||
"" if hex_tag else "Besser den vollen Git-SHA als Pin setzen (Tenant → Übersicht → Core-Version) – dann prüft der Health-Check die Core-Version.")
|
||||
if source == "admin":
|
||||
add("bundle", "ok" if bundle_ok else "fail", "Core kann Bundle laden", "Einstellung core_supports_bundle", "" if bundle_ok else "Einstellungen → „Core kann Bundle laden“ aktivieren (erst wenn das Image den Bundle-Loader enthält).")
|
||||
good_url = admin_url.startswith("https://") and "<" not in admin_url
|
||||
add("adminurl", "ok" if good_url else "fail", "Öffentliche Admin-URL", admin_url or "nicht gesetzt",
|
||||
"" if good_url else "Einstellungen → „Öffentliche URL der Admin-Plattform“ (https://…): so holt der Tenant-Container sein Bundle.")
|
||||
if good_url:
|
||||
try:
|
||||
status = (self_get or _get)(f"{admin_url.rstrip('/')}/healthz")
|
||||
add("adminreach", "ok" if status == 200 else "fail", "Admin von außen erreichbar", f"GET /healthz → {status}", "" if status == 200 else "Domain/Proxy in Coolify prüfen.")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
add("adminreach", "fail", "Admin von außen erreichbar", f"{type(exc).__name__}: {str(exc)[:120]}", "Domain/Proxy in Coolify prüfen.")
|
||||
# Mittwald
|
||||
add("domain", "ok" if base_domain else "fail", "Basis-Domain", base_domain or "nicht gesetzt", "" if base_domain else "Einstellungen → Basis-Domain (= Gitea-Variable DOMAIN).")
|
||||
add("registry", "ok" if registry else "fail", "Registry", registry or "nicht gesetzt", "" if registry else "Einstellungen → Registry (= Gitea-Variable REGISTRY).")
|
||||
add("mtoken", "ok" if deploy.token_present() else "fail", "MITTWALD_API_TOKEN", "gesetzt" if deploy.token_present() else "nicht gesetzt",
|
||||
"" if deploy.token_present() else "Env MITTWALD_API_TOKEN der Admin-Plattform setzen (derselbe Token wie in Gitea).")
|
||||
add("mproject", "ok" if (project or stack) else "fail", "Mittwald PROJECT_ID / STACK_ID", f"{project or '–'} / {stack or '–'}", "" if (project or stack) else "Einstellungen → PROJECT_ID (und optional STACK_ID).")
|
||||
if deploy.token_present() and (project or stack):
|
||||
try:
|
||||
mw = Mittwald(__import__("os").getenv("MITTWALD_API_TOKEN", ""), __import__("os").getenv("MITTWALD_API_BASE", "https://api.mittwald.de/v2"), False, lambda m: None, mw_http)
|
||||
sid = mw.resolve_stack_id(project, stack)
|
||||
services = sorted(Mittwald.services_of(mw.api("GET", f"/stacks/{sid}/", write=False)))
|
||||
mine = f"app-{tenant_id}"
|
||||
protected = [s for s in services if s != mine]
|
||||
add("stack", "ok", "Mittwald-Stack lesbar", f"{len(services)} Services; bestehende bleiben unangetastet: {', '.join(protected) or '–'}")
|
||||
add("svc", "info", "Service dieses Tenants", f"{mine} " + ("existiert bereits (wird aktualisiert)" if mine in services else "existiert noch nicht (wird angelegt)"))
|
||||
except (DeployError, DryRunSkip) as exc:
|
||||
add("stack", "fail", "Mittwald-Stack lesbar", str(exc)[:200], "Token-Rechte, PROJECT_ID/STACK_ID prüfen.")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
add("stack", "fail", "Mittwald-Stack lesbar", f"{type(exc).__name__}: {str(exc)[:160]}", "Netzwerk/Token prüfen.")
|
||||
# Secrets / OIDC / STT
|
||||
will_create = oc["mode"] == "managed" and onyva_login.key_present() and not has_oidc_secrets
|
||||
real_missing = [m for m in missing if not (will_create and m in ("OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET"))]
|
||||
add("secrets", "ok" if not real_missing else "fail", "Pflicht-Secrets", "vollständig" if not real_missing else "fehlen: " + ", ".join(real_missing),
|
||||
"" if not real_missing else "Tenant → Secrets (tenantspezifisch) bzw. Einstellungen → Geteilte Secrets (z. B. CORTECS_API_KEY).")
|
||||
if will_create:
|
||||
add("oidcnew", "info", "OIDC-Client", "wird beim Deploy automatisch im Login angelegt (Client-ID/-Secret sofort gespeichert)")
|
||||
elif oc["mode"] != "off" and not onyva_login.key_present():
|
||||
add("oidckey", "fail", "ONYVA_LOGIN_API_KEY", "nicht gesetzt – OIDC-Abgleich beim Deploy nicht möglich", "Env ONYVA_LOGIN_API_KEY setzen oder OIDC-Verwaltung auf „off“ stellen und Client manuell eintragen.")
|
||||
elif oc["mode"] != "off":
|
||||
add("oidckey", "ok", "Login-API", f"Modus {oc['mode']}")
|
||||
if oc["mode"] != "off" and oc["auth_mode"] == "required" and not oc["members"]:
|
||||
add("members", "warn", "Freigaben", "auth_mode „required“, aber keine Nutzer gepflegt – niemand kann sich anmelden",
|
||||
"Domain & Login → Freigaben: mindestens dich eintragen (Nutzer muss im Login existieren).")
|
||||
if stt_provider != "cortecs":
|
||||
add("stt", "info", "STT-Provider", stt_provider + " – braucht MISTRAL_API_KEY")
|
||||
# Inhalt
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tenant_id)
|
||||
try:
|
||||
val = preview.run(db, t, "current", "validate")
|
||||
add("render", "ok" if val.get("ok") else "fail", "Veröffentlichter Stand rendert (echter Core)", "; ".join(val.get("errors", []))[:200] or "Web-UI und PDF ok",
|
||||
"" if val.get("ok") else "Tenant → Editor korrigieren und neu veröffentlichen.")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
add("render", "fail", "Rendering", str(exc)[-200:])
|
||||
if draft:
|
||||
add("draft", "warn", "Unveröffentlichter Entwurf", "wird NICHT mit deployt", "Erst veröffentlichen, wenn er mit soll.")
|
||||
add("host", "info", "Ziel-URL", f"https://{host}", "")
|
||||
ready = not any(i["level"] == "fail" for i in out)
|
||||
return {"ready": ready, "items": out, "host": host, "tag": tag}
|
||||
|
||||
|
||||
def _get(url: str) -> int:
|
||||
import httpx
|
||||
return httpx.get(url, timeout=8, follow_redirects=True).status_code
|
||||
109
app/services/preview.py
Normal file
109
app/services/preview.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""Live-Vorschau: führt den vendored Core mit dem (Entwurfs-)Tenant in einem Subprozess aus."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Tenant
|
||||
from app.services import store
|
||||
from app.services.samples import DEFAULT_SAMPLE
|
||||
from app.settings import PLATFORM_DIR, PREVIEW_CACHE
|
||||
|
||||
WORKER = Path(__file__).with_name("preview_worker.py")
|
||||
TIMEOUT = 90
|
||||
|
||||
|
||||
class PreviewError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def core_version() -> str:
|
||||
from app.services import contract
|
||||
return contract.sha()[:10]
|
||||
|
||||
|
||||
def sample_for(t: Tenant) -> str:
|
||||
return t.preview_sample or DEFAULT_SAMPLE
|
||||
|
||||
|
||||
EXT = {"index": "html", "pdf": "pdf", "validate": "json", "prompt": "json"}
|
||||
|
||||
|
||||
class SourceGone(LookupError):
|
||||
pass
|
||||
|
||||
|
||||
def resolve_content(db: Session, t: Tenant, source: str) -> tuple[dict, dict]:
|
||||
"""draft | current | v<N> | ws:<run> (Live-Arbeitsstand des Agenten) | proposal:<msg> (Vorschlag des Agenten)."""
|
||||
if source.startswith("ws:"):
|
||||
from app.agent.loop import WORKSPACES
|
||||
if source[3:] not in WORKSPACES:
|
||||
raise SourceGone("Der Arbeitsstand des Agenten ist nicht mehr im Speicher (Neustart?). Bitte „Entwurf“ wählen.")
|
||||
cfg, files = WORKSPACES[source[3:]]
|
||||
return dict(cfg), dict(files)
|
||||
if source.startswith("proposal:"):
|
||||
from app.agent.loop import proposal_content
|
||||
from app.models import ChatMessage
|
||||
m = db.get(ChatMessage, int(source[9:] or 0))
|
||||
if m is None or m.tenant_id != t.id or not m.proposal:
|
||||
raise SourceGone("Vorschlag nicht gefunden.")
|
||||
return proposal_content(db, t, m.proposal)
|
||||
return store.content(db, t, source)
|
||||
|
||||
|
||||
def run(db: Session, t: Tenant, source: str, kind: str, sample: str | None = None, args: dict | None = None) -> bytes | dict:
|
||||
cfg, files = resolve_content(db, t, source)
|
||||
return run_content(db, t, cfg, files, kind, sample, args)
|
||||
|
||||
|
||||
def run_content(db: Session, t: Tenant, cfg: dict, files: dict, kind: str, sample: str | None = None,
|
||||
args: dict | None = None) -> bytes | dict:
|
||||
"""Rendert beliebigen Inhalt (cfg + {pfad: sha}) – für Entwurf, Version oder Agent-Workspace."""
|
||||
sample = sample if sample is not None else sample_for(t)
|
||||
h = hashlib.sha256(json.dumps([cfg, files, kind, sample, args or {}, core_version()], sort_keys=True).encode())
|
||||
cached = PREVIEW_CACHE / f"{h.hexdigest()[:32]}.{EXT[kind]}"
|
||||
if not cached.exists():
|
||||
_execute(db, t.id, cfg, files, kind, sample, cached, args or {})
|
||||
data = cached.read_bytes()
|
||||
return json.loads(data) if EXT[kind] == "json" else data
|
||||
|
||||
|
||||
def _limits() -> None:
|
||||
import resource
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (90, 90))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (200 * 1024 * 1024,) * 2)
|
||||
os.setsid()
|
||||
|
||||
|
||||
def _execute(db: Session, tenant_id: str, cfg: dict, files: dict, kind: str, sample: str, out: Path, args: dict) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="onyscribe-preview-") as tmp_s:
|
||||
tmp = Path(tmp_s)
|
||||
shutil.copytree(PLATFORM_DIR / "core", tmp / "core", ignore=shutil.ignore_patterns("__pycache__"))
|
||||
store.write_tenant_dir(db, cfg, files, tmp / "tenants" / tenant_id)
|
||||
(tmp / "data").mkdir()
|
||||
shutil.copy(WORKER, tmp / "preview_worker.py")
|
||||
env = {
|
||||
"PATH": os.environ.get("PATH", ""), "HOME": str(tmp), "LANG": "C.UTF-8",
|
||||
"TENANT": tenant_id, "DATA_DIR": str(tmp / "data"), "AUTH_ENABLED": "false",
|
||||
"PYTHONPATH": str(tmp), "PYTHONDONTWRITEBYTECODE": "1",
|
||||
# Kein einziges echtes Secret im Sandbox-Prozess
|
||||
"OIDC_ISSUER": "https://login.example.invalid", "SESSION_SECRET": "preview",
|
||||
}
|
||||
result = tmp / f"result.{out.suffix.lstrip('.')}"
|
||||
payload = json.dumps({"summary": sample, "title": f"{cfg.get('name', tenant_id)} – Vorschau", **args})
|
||||
try:
|
||||
proc = subprocess.run([sys.executable, "-I", "preview_worker.py", kind, str(result)], input=payload,
|
||||
cwd=tmp, env=env, capture_output=True, text=True, timeout=TIMEOUT, preexec_fn=_limits)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise PreviewError("Vorschau-Timeout (Tenant-Code läuft zu lange)") from exc
|
||||
if proc.returncode != 0 or not result.exists():
|
||||
raise PreviewError((proc.stderr or proc.stdout or "unbekannter Fehler")[-2500:])
|
||||
shutil.copy(result, out)
|
||||
65
app/services/preview_worker.py
Normal file
65
app/services/preview_worker.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""Läuft in einem isolierten Verzeichnis mit Kopie des Cores + materialisiertem Tenant.
|
||||
|
||||
Aufruf: python preview_worker.py <index|pdf|validate> <ausgabedatei> (JSON-Eingabe auf stdin)
|
||||
Führt den ECHTEN Core aus (auch overrides.py) – deshalb nur als Subprozess ohne Secrets/Netz-Keys.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
sys.path.insert(0, os.getcwd())
|
||||
|
||||
kind, out = sys.argv[1], sys.argv[2]
|
||||
args = json.loads(sys.stdin.read() or "{}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
from core import app as appmod
|
||||
from core.config import CFG
|
||||
|
||||
if kind == "index":
|
||||
page = appmod._render_index()
|
||||
open(out, "w", encoding="utf-8").write(page)
|
||||
elif kind == "pdf":
|
||||
pdf = appmod.build_pdf(args.get("title") or "Vorschau", args["summary"], "SPRECHER_0: Beispiel.")
|
||||
open(out, "wb").write(pdf)
|
||||
elif kind == "prompt":
|
||||
base = appmod._summary_prompt()
|
||||
ctx = {"title": args.get("title", ""), "insurance_type": args.get("terminart", ""),
|
||||
"customer_2": args.get("projekt", ""), "customer_name": args.get("kunde", ""), "customer_address": ""}
|
||||
final = appmod.HOOKS.build_prompt(base, ctx)
|
||||
json.dump({"prompt": final, "chars": len(final)}, open(out, "w"))
|
||||
elif kind == "validate":
|
||||
errors, warnings = [], []
|
||||
page = appmod._render_index()
|
||||
if "@@" in page:
|
||||
errors.append("Web-UI: nicht ersetzte @@PLATZHALTER@@ im Template")
|
||||
if not CFG.logo_path:
|
||||
warnings.append("Kein Logo (logo.svg/logo.png) hinterlegt")
|
||||
if not CFG.favicon_path:
|
||||
warnings.append("Kein favicon.svg – es wird ein generisches Favicon verwendet")
|
||||
try:
|
||||
pdf = appmod.build_pdf("Validierung", args["summary"], "SPRECHER_0: Beispiel.")
|
||||
if pdf[:5] != b"%PDF-":
|
||||
errors.append("PDF-Rendering lieferte kein PDF")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(f"PDF-Rendering fehlgeschlagen: {exc}")
|
||||
prompt = CFG.prompt_path.read_text(encoding="utf-8")
|
||||
if len(prompt.strip()) < 200:
|
||||
warnings.append("Prompt ist sehr kurz")
|
||||
json.dump({"ok": not errors, "errors": errors, "warnings": warnings,
|
||||
"tenant": CFG.id, "prompt_chars": len(prompt)}, open(out, "w"))
|
||||
|
||||
|
||||
try:
|
||||
main()
|
||||
except SystemExit:
|
||||
raise
|
||||
except BaseException: # noqa: BLE001
|
||||
tb = traceback.format_exc()
|
||||
if kind == "validate":
|
||||
json.dump({"ok": False, "errors": ["Tenant lässt sich nicht laden:\n" + tb[-1500:]], "warnings": []}, open(out, "w"))
|
||||
else:
|
||||
sys.stderr.write(tb)
|
||||
sys.exit(2)
|
||||
254
app/services/releases.py
Normal file
254
app/services/releases.py
Normal file
@@ -0,0 +1,254 @@
|
||||
"""Core-Releases: Eingang (CI), Kompatibilitätsprüfung aller Tenants, Freigabe, Canary-Rollout, Rollback.
|
||||
|
||||
Ablauf: CI meldet Image-Tag → Admin (läuft auf genau diesem Core) prüft alle Tenants in der Sandbox → Freigabe →
|
||||
Rollout (Canary zuerst, Abbruch beim ersten Fehler, Health-Check pro Tenant) → aktiver Tag wird der neue Standard.
|
||||
Gepinnte Tenants nehmen nicht teil. Echte Rollouts brauchen DEPLOY_LIVE=true; sonst nur Dry-Run.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import re
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import session
|
||||
from app.models import CoreRelease, Deployment, Setting, Tenant
|
||||
from app.services import contract, deploy, guard, preview, store
|
||||
|
||||
DATE = re.compile(r"\b\d{1,2}\.\d{1,2}\.\d{2,4}\b")
|
||||
|
||||
|
||||
class ReleaseError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
# ── Eingang ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def announce(db, sha: str, tag: str | None, ref: str, image: str, message: str) -> tuple[CoreRelease, bool]:
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{7,64}", sha or ""):
|
||||
raise ReleaseError("sha: 7–64 Hex-Zeichen erwartet")
|
||||
sha = sha.lower()
|
||||
r = db.scalar(select(CoreRelease).where(CoreRelease.sha == sha))
|
||||
created = r is None
|
||||
if created:
|
||||
r = CoreRelease(sha=sha, tag=(tag or sha)[:100], ref=ref[:100], image=image[:300], message=message[:500])
|
||||
db.add(r)
|
||||
store.audit(db, "ci", "core.release", None, f"{sha[:10]} ({ref}) {message[:80]}")
|
||||
else:
|
||||
r.message, r.ref, r.image = message[:500] or r.message, ref[:100] or r.ref, image[:300] or r.image
|
||||
db.flush()
|
||||
return r, created
|
||||
|
||||
|
||||
def admin_matches(r: CoreRelease) -> bool:
|
||||
return contract.same_sha(contract.sha(), r.sha)
|
||||
|
||||
|
||||
def view(r: CoreRelease, active_tag: str) -> dict:
|
||||
return {"id": r.id, "sha": r.sha, "tag": r.tag, "ref": r.ref, "image": r.image, "message": r.message, "status": r.status,
|
||||
"announced_at": r.announced_at.isoformat() if r.announced_at else None, "admin_matches": admin_matches(r),
|
||||
"approved_by": r.approved_by, "compat": r.compat, "rollout_batch": r.rollout_batch, "previous_tag": r.previous_tag,
|
||||
"active": r.tag == active_tag or contract.same_sha(r.sha, active_tag)}
|
||||
|
||||
|
||||
# ── Kompatibilitätsprüfung ──────────────────────────────────────────────────
|
||||
|
||||
def _pdf_fingerprint(pdf: bytes) -> tuple[int, str]:
|
||||
from pypdf import PdfReader
|
||||
rd = PdfReader(io.BytesIO(pdf))
|
||||
text = DATE.sub("<datum>", "\n".join((p.extract_text() or "") for p in rd.pages)) # das Tagesdatum steht im PDF
|
||||
return len(rd.pages), hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def fingerprint(db, t: Tenant) -> dict:
|
||||
"""Rendert den veröffentlichten Stand mit dem aktuellen Admin-Core und bildet Fingerabdrücke."""
|
||||
cfg, files = store.content(db, t, "current")
|
||||
sample = preview.sample_for(t)
|
||||
fp: dict = {"sample": hashlib.sha256(sample.encode()).hexdigest()[:12]}
|
||||
val = preview.run_content(db, t, cfg, files, "validate", sample)
|
||||
fp["validate"] = {"ok": val.get("ok", False), "errors": val.get("errors", []), "warnings": val.get("warnings", [])}
|
||||
if val.get("ok"):
|
||||
pages, text_sha = _pdf_fingerprint(preview.run_content(db, t, cfg, files, "pdf", sample))
|
||||
fp.update(pdf_pages=pages, pdf_text=text_sha,
|
||||
web=hashlib.sha256(preview.run_content(db, t, cfg, files, "index", sample)).hexdigest(),
|
||||
prompt=hashlib.sha256(preview.run_content(db, t, cfg, files, "prompt", sample, {"title": "x"})["prompt"].encode()).hexdigest())
|
||||
fp["schema_problems"] = contract.check_tenant_json(cfg)
|
||||
return fp
|
||||
|
||||
|
||||
def compare(fp: dict, baseline: dict | None, version: int) -> dict:
|
||||
"""→ {status: ok|changed|new|error, changed:[…], errors:[…], warnings:[…]}"""
|
||||
out = {"status": "ok", "changed": [], "errors": fp["validate"]["errors"], "warnings": list(fp["validate"]["warnings"]) + fp.get("schema_problems", [])}
|
||||
if not fp["validate"]["ok"]:
|
||||
out["status"] = "error"
|
||||
return out
|
||||
if not baseline or baseline.get("version") != version or baseline["fp"].get("sample") != fp["sample"]:
|
||||
out["status"] = "new"
|
||||
return out
|
||||
labels = {"pdf_pages": "PDF-Seitenzahl", "pdf_text": "PDF-Text", "web": "Web-UI", "prompt": "Finaler Prompt"}
|
||||
out["changed"] = [labels[k] for k in labels if baseline["fp"].get(k) != fp.get(k)]
|
||||
if out["changed"]:
|
||||
out["status"] = "changed"
|
||||
return out
|
||||
|
||||
|
||||
def check(release_id: int, *, force: bool = False, wait: bool = False, workers: int = 3) -> None:
|
||||
"""Prüft ALLE Tenants (außer archivierten) mit dem Core, auf dem die Admin-Plattform läuft. Ergebnis in release.compat."""
|
||||
with session() as db:
|
||||
r = db.get(CoreRelease, release_id)
|
||||
if r is None:
|
||||
raise ReleaseError("Release nicht gefunden")
|
||||
if not force and not admin_matches(r):
|
||||
raise ReleaseError(f"Die Admin-Plattform läuft noch auf Core {contract.sha()[:10]}, das Release ist {r.sha[:10]}. "
|
||||
"Erst den Admin neu bauen (FROM neues Core-Image) – sonst würde gegen den falschen Core geprüft.")
|
||||
ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status != "archived").order_by(Tenant.id))]
|
||||
r.compat = {"checked_at": None, "core_sha": contract.sha(), "running": True, "tenants": {}, "summary": {}, "forced": force and not admin_matches(r)}
|
||||
r.status = "new" if r.status in ("new", "checked") else r.status
|
||||
|
||||
def one(tid: str):
|
||||
with session() as db:
|
||||
t = db.get(Tenant, tid)
|
||||
try:
|
||||
fp = fingerprint(db, t)
|
||||
res = compare(fp, t.render_baseline, t.current_version)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
fp, res = {}, {"status": "error", "changed": [], "errors": [f"{type(exc).__name__}: {str(exc)[-300:]}"], "warnings": []}
|
||||
return tid, fp, res
|
||||
|
||||
def work():
|
||||
results = {}
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
for tid, fp, res in ex.map(one, ids):
|
||||
results[tid] = {**res, "fp": fp}
|
||||
summary = {k: sum(1 for v in results.values() if v["status"] == k) for k in ("ok", "changed", "new", "error")}
|
||||
with session() as db:
|
||||
r = db.get(CoreRelease, release_id)
|
||||
r.compat = {"checked_at": datetime.now(timezone.utc).isoformat(), "core_sha": contract.sha(), "running": False, "tenants": results,
|
||||
"summary": summary, "forced": bool(r.compat and r.compat.get("forced"))}
|
||||
if r.status == "new":
|
||||
r.status = "checked"
|
||||
store.audit(db, "system", "core.check", None, f"{r.sha[:10]}: {summary}")
|
||||
|
||||
th = threading.Thread(target=work, daemon=True)
|
||||
th.start()
|
||||
if wait:
|
||||
th.join()
|
||||
|
||||
|
||||
# ── Freigabe & Rollout ──────────────────────────────────────────────────────
|
||||
|
||||
def approve(db, release_id: int, actor: str, acknowledge: bool = False) -> CoreRelease:
|
||||
r = db.get(CoreRelease, release_id)
|
||||
if r is None:
|
||||
raise ReleaseError("Release nicht gefunden")
|
||||
c = r.compat
|
||||
if not c or c.get("running"):
|
||||
raise ReleaseError("Erst die Kompatibilitätsprüfung abschließen.")
|
||||
if not c.get("forced") and not contract.same_sha(c.get("core_sha", ""), r.sha):
|
||||
raise ReleaseError("Die Prüfung stammt von einem anderen Core als dieses Release – bitte erneut prüfen.")
|
||||
if c["summary"].get("error"):
|
||||
bad = [t for t, v in c["tenants"].items() if v["status"] == "error"]
|
||||
raise ReleaseError(f"Nicht freigebbar: {len(bad)} Tenant(s) mit Fehlern ({', '.join(bad)}).")
|
||||
if c["summary"].get("changed") and not acknowledge:
|
||||
raise ReleaseError("Bei einigen Tenants hat sich das Rendering geändert – bitte prüfen und bewusst bestätigen (acknowledge).")
|
||||
r.status, r.approved_by, r.approved_at = "approved", actor, datetime.now(timezone.utc)
|
||||
store.audit(db, actor, "core.approve", None, r.sha[:10])
|
||||
return r
|
||||
|
||||
|
||||
def canary_list(db) -> list[str]:
|
||||
s = db.get(Setting, "canary_tenants")
|
||||
return [x.strip() for x in (s.value if s and s.value else "demo").split(",") if x.strip()]
|
||||
|
||||
|
||||
def _set_setting(db, key: str, value: str) -> None:
|
||||
s = db.get(Setting, key)
|
||||
if s is None:
|
||||
s = Setting(key=key)
|
||||
db.add(s)
|
||||
s.value = value
|
||||
|
||||
|
||||
def rollout(release_id: int, actor: str, *, dry_run: bool, http=None, oidc_client=None, health=None, wait: bool = False) -> tuple[str, list[int]]:
|
||||
live = not dry_run
|
||||
if live and not deploy.live_enabled():
|
||||
raise ReleaseError("Echte Rollouts sind gesperrt (DEPLOY_LIVE != true). Dry-Run ist möglich.")
|
||||
with session() as db:
|
||||
r = db.get(CoreRelease, release_id)
|
||||
if r is None:
|
||||
raise ReleaseError("Release nicht gefunden")
|
||||
allowed = {"approved", "active", "failed", "rolled_back"} | ({"new", "checked"} if dry_run else set()) # Dry-Run schon vor der Freigabe
|
||||
if r.status not in allowed:
|
||||
raise ReleaseError(f"Release ist im Status „{r.status}“ – erst prüfen und freigeben.")
|
||||
tag, sha = r.tag, r.sha
|
||||
ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status == "active").order_by(Tenant.id)) if not t.core_pin]
|
||||
if live: # Live: nur für Live-Schreibzugriffe freigegebene Tenants (guard.py)
|
||||
ids = [t for t in ids if guard.live_allowed(t)]
|
||||
if not ids:
|
||||
raise ReleaseError(f"Kein Tenant für Live-Rollouts freigegeben (DEPLOY_ALLOWED_TENANTS = {guard.describe()}).")
|
||||
canary = canary_list(db)
|
||||
batch = uuid.uuid4().hex[:10]
|
||||
if live:
|
||||
r.status, r.previous_tag, r.rollout_batch = "rolling", deploy.setting(db, "core_image_tag", "latest"), batch
|
||||
store.audit(db, actor, "core.rollout", None, f"{sha[:10]} → {len(ids)} Tenants")
|
||||
|
||||
def done(ok: bool, results: dict):
|
||||
if not live:
|
||||
return
|
||||
with session() as db:
|
||||
r = db.get(CoreRelease, release_id)
|
||||
if ok:
|
||||
r.status = "active"
|
||||
_set_setting(db, "core_image_tag", tag)
|
||||
for other in db.scalars(select(CoreRelease).where(CoreRelease.status == "active", CoreRelease.id != release_id)):
|
||||
other.status = "superseded"
|
||||
for tid in results: # Baselines für künftige Kompatibilitätsprüfungen
|
||||
t = db.get(Tenant, tid)
|
||||
c = ((r.compat or {}).get("tenants") or {}).get(tid)
|
||||
if t and c and c.get("fp"):
|
||||
t.render_baseline = {"version": t.current_version, "core": sha, "fp": c["fp"]}
|
||||
else:
|
||||
r.status = "failed"
|
||||
store.audit(db, "system", "core.rollout.done", None, f"{sha[:10]} {'ok' if ok else 'FEHLER'} {results}")
|
||||
|
||||
dep_ids = deploy.run_sequence(ids, "dry-run" if dry_run else "deploy", actor, batch=batch, core_tag=tag, http=http, oidc_client=oidc_client,
|
||||
health=health, canary=canary, wait=wait, on_done=done)
|
||||
return batch, dep_ids
|
||||
|
||||
|
||||
def rollback_release(release_id: int, actor: str, *, http=None, oidc_client=None, health=None, wait: bool = False) -> tuple[str, list[int]]:
|
||||
"""Alle (nicht gepinnten) Tenants auf den Tag vor diesem Rollout zurück."""
|
||||
if not deploy.live_enabled():
|
||||
raise ReleaseError("Echte Rollbacks sind gesperrt (DEPLOY_LIVE != true).")
|
||||
with session() as db:
|
||||
r = db.get(CoreRelease, release_id)
|
||||
if r is None or not r.previous_tag:
|
||||
raise ReleaseError("Kein vorheriger Tag bekannt – Rollback nicht möglich.")
|
||||
prev = r.previous_tag
|
||||
ids = [t.id for t in db.scalars(select(Tenant).where(Tenant.status == "active").order_by(Tenant.id)) if not t.core_pin and guard.live_allowed(t.id)]
|
||||
batch = uuid.uuid4().hex[:10]
|
||||
store.audit(db, actor, "core.rollback", None, f"{r.sha[:10]} → :{prev}")
|
||||
|
||||
def done(ok, results):
|
||||
with session() as db:
|
||||
r = db.get(CoreRelease, release_id)
|
||||
if ok:
|
||||
r.status = "rolled_back"
|
||||
_set_setting(db, "core_image_tag", prev)
|
||||
store.audit(db, "system", "core.rollback.done", None, f"{'ok' if ok else 'FEHLER'} {results}")
|
||||
|
||||
return batch, deploy.run_sequence(ids, "deploy", actor, batch=batch, core_tag=prev, http=http, oidc_client=oidc_client, health=health, wait=wait, on_done=done)
|
||||
|
||||
|
||||
def tenant_rollback_tag(db, t: Tenant) -> str | None:
|
||||
"""Letzter erfolgreich live deployter Tag dieses Tenants, der vom aktuellen abweicht."""
|
||||
for d in db.scalars(select(Deployment).where(Deployment.tenant_id == t.id, Deployment.kind == "deploy", Deployment.status == "success",
|
||||
Deployment.simulated.is_(False)).order_by(Deployment.id.desc())):
|
||||
if d.image_tag and d.image_tag != t.deployed_image:
|
||||
return d.image_tag
|
||||
return None
|
||||
76
app/services/samples.py
Normal file
76
app/services/samples.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""Beispielprotokolle für die PDF-Vorschau (so, wie der LLM sie liefern würde)."""
|
||||
|
||||
DEFAULT_SAMPLE = """===TERMINART===
|
||||
Kick-off
|
||||
===/TERMINART===
|
||||
===PROJEKT===
|
||||
Muster GmbH
|
||||
Relaunch 2026
|
||||
===/PROJEKT===
|
||||
===KONTAKT===
|
||||
Max Mustermann - Geschäftsführer
|
||||
Erika Beispiel - Projektleitung
|
||||
===/KONTAKT===
|
||||
Anlass & Kontext
|
||||
Erstes Kennenlernen und Abstecken des Projektrahmens. Ziel ist ein gemeinsames Verständnis von Umfang, Zeitplan und Zuständigkeiten.
|
||||
Besprochene Themen
|
||||
• Zeitplan und Meilensteine bis zum Go-live im Herbst
|
||||
• Budgetrahmen und Freigabeprozess
|
||||
• Anbindung des bestehenden CRM
|
||||
Entscheidungen
|
||||
• Projektstart im Juli
|
||||
• Wöchentlicher Jour fixe, dienstags 10:00 Uhr
|
||||
Action Items
|
||||
• Angebot bis Freitag senden [Verantwortlich: Erika Beispiel, Frist: 18.07.]
|
||||
• Zugänge zum CRM bereitstellen [Verantwortlich: Max Mustermann, Frist: 22.07.]
|
||||
Offene Punkte
|
||||
• Finale Designrichtung noch nicht entschieden
|
||||
• Datenschutzfreigabe der Fachabteilung steht aus
|
||||
Nächste Schritte
|
||||
• Folgetermin zum Konzept-Review vereinbaren
|
||||
"""
|
||||
|
||||
ROTARY_SAMPLE = """===TERMINART===
|
||||
Clubmeeting
|
||||
===/TERMINART===
|
||||
===PROJEKT===
|
||||
Lieferketten im Wandel
|
||||
===/PROJEKT===
|
||||
Begrüßung
|
||||
Der Präsident eröffnet das Meeting und begrüßt die anwesenden Freunde sowie zwei Gäste.
|
||||
Geburtstage
|
||||
• Anna Beispiel
|
||||
• Peter Muster
|
||||
Aktuelle 5 Minuten
|
||||
Kurzer Bericht zum Stand der Spendenaktion für das Jugendprojekt. Bisher sind 4.200 Euro zusammengekommen.
|
||||
Happy Euro
|
||||
• Für die gelungene Veranstaltung am Samstag
|
||||
• Für das gute Wetter beim Grillfest
|
||||
Vortrag / Referat
|
||||
Globalisierung – historische Einordnung
|
||||
Der Referent ordnet die Entwicklung der globalen Lieferketten seit den 1990er-Jahren ein.
|
||||
Folgen für die Region
|
||||
• Abhängigkeit von wenigen Zulieferern
|
||||
• Chancen durch regionale Wertschöpfung
|
||||
Dank
|
||||
Der Präsident dankt dem Referenten für den anregenden Vortrag.
|
||||
Abschluss
|
||||
Nächstes Meeting am kommenden Montag.
|
||||
"""
|
||||
|
||||
SAMPLE_TRANSCRIPT = """SPRECHER_0: Guten Morgen zusammen, schön, dass wir uns heute zum Kick-off treffen. Ich bin Erika Beispiel, Projektleitung auf unserer Seite.
|
||||
SPRECHER_1: Morgen. Max Mustermann, ich bin Geschäftsführer der Muster GmbH. Wir wollen unsere Website bis zum Herbst neu aufsetzen.
|
||||
SPRECHER_0: Gut. Dann starten wir im Juli. Ich würde einen wöchentlichen Jour fixe vorschlagen, dienstags um zehn.
|
||||
SPRECHER_1: Passt. Das Budget liegt bei etwa vierzigtausend Euro, die Freigabe läuft über mich.
|
||||
SPRECHER_0: Ich schicke Ihnen bis Freitag das Angebot. Und wir brauchen von Ihnen die Zugänge zum CRM, möglichst bis zum zweiundzwanzigsten Juli.
|
||||
SPRECHER_1: Mache ich. Offen ist noch die Designrichtung und ob die Datenschutzabteilung zustimmt.
|
||||
SPRECHER_0: Dann vereinbaren wir als Nächstes einen Termin zum Konzept-Review.
|
||||
"""
|
||||
|
||||
ROTARY_TRANSCRIPT = """SPRECHER_0: Liebe Freunde, ich begrüße euch zum Clubmeeting und freue mich über zwei Gäste heute Abend.
|
||||
SPRECHER_0: Geburtstag haben diese Woche Anna Beispiel und Peter Muster, herzlichen Glückwunsch.
|
||||
SPRECHER_1: Zur Spendenaktion für das Jugendprojekt: Wir haben bisher viertausendzweihundert Euro zusammen.
|
||||
SPRECHER_2: Happy Euro für das schöne Wetter beim Grillfest und für die gelungene Veranstaltung am Samstag.
|
||||
SPRECHER_3: Mein Vortrag heute: Globalisierung und Lieferketten. Seit den Neunzigern hat sich die Abhängigkeit von wenigen Zulieferern stark erhöht. Für unsere Region bedeutet das Risiken, aber auch Chancen durch regionale Wertschöpfung.
|
||||
SPRECHER_0: Vielen Dank für den Vortrag. Nächstes Meeting ist am kommenden Montag.
|
||||
"""
|
||||
54
app/services/seed.py
Normal file
54
app/services/seed.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""Seed-Export/-Import: Startdaten (importierte Tenants) als Datei im Repo.
|
||||
|
||||
So startet eine frische Instanz (z. B. Coolify) mit den aktuellen Tenants, ohne Zugriff auf das alte Git-Repo.
|
||||
Secrets-WERTE sind nie enthalten – nur die Schlüsselnamen.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Blob, Secret, Setting, Tenant, Version
|
||||
|
||||
COLS = ["id", "name", "status", "source", "subdomain", "custom_domain", "domain_status", "cpu", "memory",
|
||||
"current_version", "deployed_version", "deployed_image", "preview_sample", "oidc", "notes"]
|
||||
|
||||
|
||||
def export(db: Session, path: Path) -> None:
|
||||
tenants = db.scalars(select(Tenant).order_by(Tenant.id)).all()
|
||||
shas = set()
|
||||
out = {"tenants": [], "versions": [], "secrets": [], "settings": {}}
|
||||
for t in tenants:
|
||||
out["tenants"].append({c: getattr(t, c) for c in COLS})
|
||||
for v in db.scalars(select(Version).where(Version.tenant_id == t.id).order_by(Version.number)):
|
||||
out["versions"].append({"tenant_id": t.id, "number": v.number, "message": v.message, "author": v.author,
|
||||
"config": v.config, "files": v.files})
|
||||
shas |= set(v.files.values())
|
||||
for s in db.scalars(select(Secret)):
|
||||
out["secrets"].append({"tenant_id": s.tenant_id, "key": s.key})
|
||||
out["settings"] = {s.key: s.value for s in db.scalars(select(Setting))}
|
||||
out["blobs"] = {sha: {"b64": base64.b64encode(db.get(Blob, sha).content).decode(), "text": db.get(Blob, sha).is_text} for sha in shas}
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(gzip.compress(json.dumps(out).encode(), 9))
|
||||
|
||||
|
||||
def load(db: Session, path: Path) -> int:
|
||||
data = json.loads(gzip.decompress(path.read_bytes()))
|
||||
for sha, b in data["blobs"].items():
|
||||
raw = base64.b64decode(b["b64"])
|
||||
db.add(Blob(sha=sha, content=raw, size=len(raw), is_text=b["text"]))
|
||||
for t in data["tenants"]:
|
||||
db.add(Tenant(**t))
|
||||
db.flush()
|
||||
for v in data["versions"]:
|
||||
db.add(Version(**v))
|
||||
for s in data["secrets"]:
|
||||
db.add(Secret(tenant_id=s["tenant_id"], key=s["key"], value_enc=None, source="sops-import"))
|
||||
for k, v in data["settings"].items():
|
||||
db.add(Setting(key=k, value=v))
|
||||
return len(data["tenants"])
|
||||
61
app/services/status.py
Normal file
61
app/services/status.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""Zustandsabgleich: was läuft wirklich? (Vertrag: GET /health → core.version, bundle.source/version/stale)"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import session
|
||||
from app.models import Tenant
|
||||
from app.services import contract, deploy
|
||||
|
||||
_CACHE: dict = {"ts": 0.0, "data": None}
|
||||
TTL = 30.0
|
||||
|
||||
|
||||
def _http_get(url: str):
|
||||
import httpx
|
||||
r = httpx.get(url, timeout=6, follow_redirects=True)
|
||||
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
|
||||
|
||||
|
||||
def probe(host: str, expect_tag: str, expect_bundle: int | None, get=_http_get) -> dict:
|
||||
out = {"host": host, "reachable": False, "core": None, "bundle": None, "drift": [], "error": None}
|
||||
try:
|
||||
status, body = get(f"https://{host}/health")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
out["error"] = f"{type(exc).__name__}: {str(exc)[:120]}"
|
||||
out["drift"].append("nicht erreichbar")
|
||||
return out
|
||||
out["reachable"] = status == 200
|
||||
if status != 200:
|
||||
out["drift"].append(f"HTTP {status}")
|
||||
core, bundle = body.get("core"), body.get("bundle")
|
||||
out["core"], out["bundle"] = core, bundle
|
||||
if core is None:
|
||||
out["drift"].append("Core ohne Vertrag (alte Version)")
|
||||
elif expect_tag and deploy._hex_tag(expect_tag) and not contract.same_sha(core.get("version", ""), expect_tag):
|
||||
out["drift"].append(f"Core {core.get('version', '')[:10]} statt {expect_tag[:10]}")
|
||||
if expect_bundle is not None:
|
||||
if not bundle:
|
||||
out["drift"].append("meldet keine Bundle-Quelle")
|
||||
elif bundle.get("source") != "admin" or bundle.get("version") != expect_bundle or bundle.get("stale"):
|
||||
out["drift"].append(f"Bundle {bundle.get('source')} v{bundle.get('version')}{' (veraltet)' if bundle.get('stale') else ''} statt admin v{expect_bundle}")
|
||||
elif bundle and bundle.get("source") not in ("git", None):
|
||||
out["drift"].append(f"Bundle-Quelle {bundle.get('source')}, erwartet git")
|
||||
return out
|
||||
|
||||
|
||||
def probe_all(get=_http_get, refresh: bool = False, workers: int = 6) -> dict:
|
||||
if not refresh and _CACHE["data"] is not None and time.time() - _CACHE["ts"] < TTL:
|
||||
return _CACHE["data"]
|
||||
with session() as db:
|
||||
items = []
|
||||
for t in db.scalars(select(Tenant).where(Tenant.status == "active").order_by(Tenant.id)):
|
||||
items.append((t.id, deploy.host_for(db, t), deploy.effective_tag(db, t), t.current_version if t.source == "admin" else None))
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
res = list(ex.map(lambda it: (it[0], probe(it[1], it[2], it[3], get)), items))
|
||||
data = {tid: r for tid, r in res}
|
||||
_CACHE.update(ts=time.time(), data=data)
|
||||
return data
|
||||
225
app/services/store.py
Normal file
225
app/services/store.py
Normal file
@@ -0,0 +1,225 @@
|
||||
"""Inhalte eines Tenants: Blobs, Entwurf, Versionen, Diff, Bundle.
|
||||
|
||||
Ein Tenant besteht aus `config` (Inhalt der tenant.json ohne deploy-Block) und
|
||||
`files` ({pfad: sha}). Bearbeitet wird immer ein Entwurf (tenant.draft);
|
||||
„Veröffentlichen“ schreibt daraus eine neue unveränderliche Version.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import tarfile
|
||||
from copy import deepcopy
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
|
||||
from app.models import AuditLog, Blob, Tenant, Version
|
||||
|
||||
TEXT_SUFFIXES = {".txt", ".html", ".css", ".py", ".json", ".svg", ".md", ".yaml", ".yml", ".csv"}
|
||||
RESERVED = {"tenant.json"} # wird aus config erzeugt
|
||||
|
||||
|
||||
def is_text_path(path: str) -> bool:
|
||||
return any(path.lower().endswith(s) for s in TEXT_SUFFIXES)
|
||||
|
||||
|
||||
def put_blob(db: Session, data: bytes, path: str = "") -> str:
|
||||
sha = hashlib.sha256(data).hexdigest()
|
||||
if db.get(Blob, sha) is None:
|
||||
db.add(Blob(sha=sha, content=data, size=len(data), is_text=is_text_path(path)))
|
||||
db.flush()
|
||||
return sha
|
||||
|
||||
|
||||
def get_blob(db: Session, sha: str) -> bytes:
|
||||
b = db.get(Blob, sha)
|
||||
if b is None:
|
||||
raise KeyError(sha)
|
||||
return b.content
|
||||
|
||||
|
||||
def audit(db: Session, actor: str, action: str, tenant_id: str | None = None, detail: str = "") -> None:
|
||||
db.add(AuditLog(actor=actor, action=action, tenant_id=tenant_id, detail=detail))
|
||||
|
||||
|
||||
def current_version(db: Session, t: Tenant) -> Version | None:
|
||||
if t.current_version is None:
|
||||
return None
|
||||
return db.scalar(select(Version).where(Version.tenant_id == t.id, Version.number == t.current_version))
|
||||
|
||||
|
||||
def get_version(db: Session, tenant_id: str, number: int) -> Version | None:
|
||||
return db.scalar(select(Version).where(Version.tenant_id == tenant_id, Version.number == number))
|
||||
|
||||
|
||||
def content(db: Session, t: Tenant, source: str = "draft") -> tuple[dict, dict]:
|
||||
"""(config, files) für source = draft | current | v<N>. Draft fällt auf current zurück."""
|
||||
if source == "draft" and t.draft:
|
||||
return deepcopy(t.draft["config"]), dict(t.draft["files"])
|
||||
if source.startswith("v") and source[1:].isdigit():
|
||||
v = get_version(db, t.id, int(source[1:]))
|
||||
else:
|
||||
v = current_version(db, t)
|
||||
if v is None:
|
||||
return {}, {}
|
||||
return deepcopy(v.config), dict(v.files)
|
||||
|
||||
|
||||
def ensure_draft(db: Session, t: Tenant) -> dict:
|
||||
"""Arbeitskopie des Entwurfs (nie das am ORM-Objekt hängende Dict mutieren)."""
|
||||
if t.draft:
|
||||
return json.loads(json.dumps(t.draft))
|
||||
cfg, files = content(db, t, "current")
|
||||
return {"config": cfg, "files": files}
|
||||
|
||||
|
||||
def _save_draft(t: Tenant, draft: dict) -> None:
|
||||
t.draft = json.loads(json.dumps(draft))
|
||||
flag_modified(t, "draft")
|
||||
|
||||
|
||||
def set_config(db: Session, t: Tenant, config: dict) -> None:
|
||||
d = ensure_draft(db, t)
|
||||
config = {k: v for k, v in config.items() if k != "deploy"}
|
||||
config["id"] = t.id
|
||||
d["config"] = config
|
||||
_save_draft(t, d)
|
||||
_sync_draft_state(db, t)
|
||||
|
||||
|
||||
def set_file(db: Session, t: Tenant, path: str, data: bytes) -> str:
|
||||
path = clean_path(path)
|
||||
d = ensure_draft(db, t)
|
||||
sha = put_blob(db, data, path)
|
||||
d["files"][path] = sha
|
||||
_save_draft(t, d)
|
||||
_sync_draft_state(db, t)
|
||||
return sha
|
||||
|
||||
|
||||
def delete_file(db: Session, t: Tenant, path: str) -> None:
|
||||
d = ensure_draft(db, t)
|
||||
d["files"].pop(path, None)
|
||||
_save_draft(t, d)
|
||||
_sync_draft_state(db, t)
|
||||
|
||||
|
||||
def clean_path(path: str) -> str:
|
||||
path = path.strip().lstrip("/")
|
||||
if not path or ".." in path.split("/") or path in RESERVED or "\\" in path:
|
||||
raise ValueError(f"Ungültiger Dateipfad: {path!r}")
|
||||
return path
|
||||
|
||||
|
||||
def _sync_draft_state(db: Session, t: Tenant) -> None:
|
||||
"""Entwurf, der identisch zur aktuellen Version ist, wieder löschen."""
|
||||
if t.draft and not diff_summary(db, t, "current", "draft")["changed"]:
|
||||
t.draft = None
|
||||
|
||||
|
||||
def discard_draft(db: Session, t: Tenant) -> None:
|
||||
t.draft = None
|
||||
|
||||
|
||||
def publish(db: Session, t: Tenant, message: str, author: str) -> Version:
|
||||
if not t.draft:
|
||||
raise ValueError("Kein Entwurf zum Veröffentlichen.")
|
||||
number = (t.current_version or 0) + 1
|
||||
last = db.scalar(select(Version.number).where(Version.tenant_id == t.id).order_by(Version.number.desc()))
|
||||
number = max(number, (last or 0) + 1)
|
||||
v = Version(tenant_id=t.id, number=number, message=message.strip() or f"Version {number}", author=author,
|
||||
config=t.draft["config"], files=t.draft["files"])
|
||||
db.add(v)
|
||||
t.current_version = number
|
||||
t.draft = None
|
||||
t.name = v.config.get("name", t.name)
|
||||
audit(db, author, "version.publish", t.id, f"v{number}: {v.message}")
|
||||
db.flush()
|
||||
return v
|
||||
|
||||
|
||||
def restore_as_draft(db: Session, t: Tenant, number: int) -> None:
|
||||
v = get_version(db, t.id, number)
|
||||
if v is None:
|
||||
raise KeyError(number)
|
||||
t.draft = {"config": deepcopy(v.config), "files": dict(v.files)}
|
||||
_sync_draft_state(db, t)
|
||||
|
||||
|
||||
# ── Diff ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def diff_summary(db: Session, t: Tenant, a: str, b: str, with_text: bool = False) -> dict:
|
||||
return diff_content(db, content(db, t, a), content(db, t, b), with_text)
|
||||
|
||||
|
||||
def diff_content(db: Session, left: tuple[dict, dict], right: tuple[dict, dict], with_text: bool = False) -> dict:
|
||||
ca, fa = left
|
||||
cb, fb = right
|
||||
items = []
|
||||
if json.dumps(ca, sort_keys=True) != json.dumps(cb, sort_keys=True):
|
||||
items.append(_item("tenant.json", "geändert", db, json.dumps(ca, indent=2, ensure_ascii=False, sort_keys=True),
|
||||
json.dumps(cb, indent=2, ensure_ascii=False, sort_keys=True), True, with_text))
|
||||
for path in sorted(set(fa) | set(fb)):
|
||||
sa, sb = fa.get(path), fb.get(path)
|
||||
if sa == sb:
|
||||
continue
|
||||
kind = "neu" if sa is None else "gelöscht" if sb is None else "geändert"
|
||||
text = is_text_path(path)
|
||||
ta = get_blob(db, sa).decode("utf-8", "replace") if (sa and text and with_text) else ""
|
||||
tb = get_blob(db, sb).decode("utf-8", "replace") if (sb and text and with_text) else ""
|
||||
items.append(_item(path, kind, db, ta, tb, text, with_text))
|
||||
return {"changed": bool(items), "items": items}
|
||||
|
||||
|
||||
def _item(path, kind, db, ta, tb, text, with_text):
|
||||
out = {"path": path, "kind": kind, "text": text}
|
||||
if with_text and text:
|
||||
lines = list(difflib.unified_diff(ta.splitlines(), tb.splitlines(), "vorher", "nachher", lineterm="", n=2))
|
||||
out["diff"] = lines
|
||||
out["added"] = sum(1 for l in lines if l.startswith("+") and not l.startswith("+++"))
|
||||
out["removed"] = sum(1 for l in lines if l.startswith("-") and not l.startswith("---"))
|
||||
return out
|
||||
|
||||
|
||||
# ── Bundle (genau das, was ein Tenant-Container beim Start abholt) ──────────
|
||||
|
||||
def tenant_json_bytes(config: dict) -> bytes:
|
||||
return (json.dumps(config, indent=2, ensure_ascii=False) + "\n").encode()
|
||||
|
||||
|
||||
def materialize(db: Session, t: Tenant, source: str, target) -> dict:
|
||||
"""Schreibt das Tenant-Verzeichnis (tenant.json + Dateien) nach `target`."""
|
||||
cfg, files = content(db, t, source)
|
||||
write_tenant_dir(db, cfg, files, target)
|
||||
return cfg
|
||||
|
||||
|
||||
def write_tenant_dir(db: Session, cfg: dict, files: dict, target) -> None:
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
(target / "tenant.json").write_bytes(tenant_json_bytes(cfg))
|
||||
for path, sha in files.items():
|
||||
p = target / path
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(get_blob(db, sha))
|
||||
|
||||
|
||||
def bundle_tar(db: Session, t: Tenant, source: str = "current") -> tuple[bytes, dict]:
|
||||
cfg, files = content(db, t, source)
|
||||
manifest = {"tenant": t.id, "version": t.current_version if source != "draft" else None,
|
||||
"files": {"tenant.json": hashlib.sha256(tenant_json_bytes(cfg)).hexdigest(), **files}}
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
||||
def add(name, data):
|
||||
ti = tarfile.TarInfo(name)
|
||||
ti.size = len(data)
|
||||
ti.mtime = 0
|
||||
tar.addfile(ti, io.BytesIO(data))
|
||||
add("tenant.json", tenant_json_bytes(cfg))
|
||||
for path, sha in sorted(files.items()):
|
||||
add(path, get_blob(db, sha))
|
||||
add("MANIFEST.json", json.dumps(manifest, indent=2).encode())
|
||||
return buf.getvalue(), manifest
|
||||
95
app/services/tenants.py
Normal file
95
app/services/tenants.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""Tenant-Lebenszyklus: anlegen, klonen, Secrets, Cut-over-Check."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from copy import deepcopy
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Deployment, Secret, Tenant, Version
|
||||
from app.services import crypto, defaults, store
|
||||
|
||||
SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{1,38}$")
|
||||
|
||||
|
||||
def create(db: Session, tenant_id: str, name: str, clone_from: str | None, actor: str) -> Tenant:
|
||||
tenant_id = tenant_id.strip().lower()
|
||||
if not SLUG.match(tenant_id):
|
||||
raise ValueError("ID: nur a-z, 0-9 und Bindestrich, 2–39 Zeichen.")
|
||||
if db.get(Tenant, tenant_id):
|
||||
raise ValueError(f"Tenant „{tenant_id}“ existiert bereits.")
|
||||
t = Tenant(id=tenant_id, name=name.strip() or tenant_id, subdomain=tenant_id, domain_status="none", source="admin",
|
||||
oidc={"mode": "managed"})
|
||||
db.add(t)
|
||||
db.flush()
|
||||
if clone_from:
|
||||
src = db.get(Tenant, clone_from)
|
||||
cfg, files = store.content(db, src, "current")
|
||||
cfg = deepcopy(cfg)
|
||||
cfg.update({"id": tenant_id, "name": t.name, "title": f"{t.name} – Gesprächsprotokoll",
|
||||
"description": f"Gesprächsprotokoll-Tool für {t.name}", "ai_detect_org": t.name, "org_short": t.name})
|
||||
# Personenbezogene/mandantenspezifische Dateien nicht mitklonen
|
||||
files = {p: s for p, s in files.items() if not p.startswith("namensliste")}
|
||||
t.cpu, t.memory = src.cpu, src.memory
|
||||
else:
|
||||
cfg, files = defaults.skeleton_config(tenant_id, t.name), {}
|
||||
db.add(Version(tenant_id=tenant_id, number=1, message="Angelegt" + (f" (Kopie von {clone_from})" if clone_from else ""),
|
||||
author=actor, config=cfg, files=files))
|
||||
t.current_version = 1
|
||||
for key, val in (("SESSION_SECRET", defaults.new_token()), ("ADMIN_BUNDLE_TOKEN", defaults.new_token())):
|
||||
db.add(Secret(tenant_id=tenant_id, key=key, value_enc=crypto.encrypt(val), source="manual"))
|
||||
for key in ("OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "COST_INGEST_KEY"):
|
||||
db.add(Secret(tenant_id=tenant_id, key=key, value_enc=None, source="manual"))
|
||||
store.audit(db, actor, "tenant.create", tenant_id, f"Kopie von {clone_from}" if clone_from else "leer")
|
||||
db.flush()
|
||||
return t
|
||||
|
||||
|
||||
def secret_rows(db: Session, tenant_id: str) -> list[dict]:
|
||||
rows = {s.key: s for s in db.scalars(select(Secret).where(Secret.tenant_id == tenant_id))}
|
||||
keys = list(dict.fromkeys(defaults.REQUIRED_SECRETS + [k for k in rows if k not in defaults.REQUIRED_SECRETS]))
|
||||
out = []
|
||||
for k in keys:
|
||||
r = rows.get(k)
|
||||
out.append({
|
||||
"key": k, "set": bool(r and r.value_enc), "source": r.source if r else "",
|
||||
"required": k in defaults.REQUIRED_SECRETS, "help": defaults.SECRET_HELP.get(k, ""),
|
||||
"updated_at": r.updated_at.isoformat() if r and r.updated_at else None,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def set_secret(db: Session, tenant_id: str, key: str, value: str | None, actor: str) -> None:
|
||||
if not re.match(r"^[A-Z][A-Z0-9_]{1,60}$", key):
|
||||
raise ValueError("Schlüssel: GROSSBUCHSTABEN_MIT_UNTERSTRICH")
|
||||
r = db.scalar(select(Secret).where(Secret.tenant_id == tenant_id, Secret.key == key))
|
||||
if r is None:
|
||||
r = Secret(tenant_id=tenant_id, key=key)
|
||||
db.add(r)
|
||||
r.value_enc = crypto.encrypt(value) if value else None
|
||||
r.source = "manual"
|
||||
store.audit(db, actor, "secret.set" if value else "secret.clear", tenant_id, key)
|
||||
|
||||
|
||||
def check_bundle_token(db: Session, tenant_id: str, token: str) -> bool:
|
||||
r = db.scalar(select(Secret).where(Secret.tenant_id == tenant_id, Secret.key == "ADMIN_BUNDLE_TOKEN"))
|
||||
return bool(r and r.value_enc and token and crypto.decrypt(r.value_enc) == token)
|
||||
|
||||
|
||||
def cutover_checks(db: Session, t: Tenant, validation: dict | None) -> list[dict]:
|
||||
secrets_ok = all(s["set"] for s in secret_rows(db, t.id) if s["required"])
|
||||
imported = db.scalar(select(Version).where(Version.tenant_id == t.id, Version.author == "import")) is not None
|
||||
dry = db.scalar(select(Deployment).where(Deployment.tenant_id == t.id, Deployment.kind == "dry-run",
|
||||
Deployment.status == "success").order_by(Deployment.id.desc()))
|
||||
return [
|
||||
{"id": "content", "label": "Inhalt aus Git importiert & Hash-gleich verifiziert" if imported else "Inhalt in der Datenbank",
|
||||
"ok": True},
|
||||
{"id": "validate", "label": "Validierung (Core-Rendering von Web-UI und PDF)", "ok": bool(validation and validation.get("ok"))},
|
||||
{"id": "secrets", "label": "Alle Pflicht-Secrets in der Admin-DB gesetzt", "ok": secrets_ok},
|
||||
{"id": "oidc", "label": "OIDC-Client abgeglichen (Login-API)" if (t.oidc or {}).get("mode", "adopt") != "off" else "OIDC-Verwaltung aus",
|
||||
"ok": (t.oidc or {}).get("mode", "adopt") == "off" or bool(((t.oidc or {}).get("last_sync") or {}).get("ok"))},
|
||||
{"id": "domain", "label": "Domain verifiziert", "ok": t.domain_status == "verified"},
|
||||
{"id": "dryrun", "label": "Dry-Run erfolgreich", "ok": dry is not None},
|
||||
{"id": "published", "label": "Keine unveröffentlichten Änderungen", "ok": not t.draft},
|
||||
]
|
||||
Reference in New Issue
Block a user