Health-Check: DNS-over-HTTPS-Rückfall bei nicht auflösbarem Hostnamen, DNS-Problem des Admin-Servers nur Warnung (außer strikte Rollouts); COST_INGEST_KEY optional (Schema-getrieben)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
RubenRWU
2026-10-15 16:36:54 +02:00
parent 31a99e31c0
commit 9c352be05b
10 changed files with 146 additions and 30 deletions

View File

@@ -17,13 +17,13 @@ COLOR_LABELS = {
} }
# Secret-Schlüssel = Env-Namen im Container (siehe infra/deploy.py) # Secret-Schlüssel = Env-Namen im Container (siehe infra/deploy.py)
REQUIRED_SECRETS = ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "COST_INGEST_KEY"] REQUIRED_SECRETS = ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET"] # Rückfall; maßgeblich ist das Schema des Cores (contract.required_secrets)
OPTIONAL_SECRETS = ["MISTRAL_API_KEY"] OPTIONAL_SECRETS = ["MISTRAL_API_KEY"]
SECRET_HELP = { SECRET_HELP = {
"OIDC_CLIENT_ID": "Client-ID aus login.onyva.dev (manuell angelegt)", "OIDC_CLIENT_ID": "Client-ID aus login.onyva.dev (manuell angelegt)",
"OIDC_CLIENT_SECRET": "Client-Secret aus login.onyva.dev", "OIDC_CLIENT_SECRET": "Client-Secret aus login.onyva.dev",
"SESSION_SECRET": "Signiert Session-Cookies – kann hier erzeugt werden", "SESSION_SECRET": "Signiert Session-Cookies – kann hier erzeugt werden",
"COST_INGEST_KEY": "Schlüssel für die Kostenerfassung (Cost-API)", "COST_INGEST_KEY": "Optional – Schlüssel für die Kostenerfassung (Cost-API); ohne Key bleibt sie inaktiv",
"MISTRAL_API_KEY": "Nur nötig bei STT-Provider „mistral“ (bei „cortecs“ wird der geteilte Key genutzt)", "MISTRAL_API_KEY": "Nur nötig bei STT-Provider „mistral“ (bei „cortecs“ wird der geteilte Key genutzt)",
"ADMIN_BUNDLE_TOKEN": "Token, mit dem der Container sein Bundle bei der Admin-Plattform abholt", "ADMIN_BUNDLE_TOKEN": "Token, mit dem der Container sein Bundle bei der Admin-Plattform abholt",
} }

View File

@@ -16,7 +16,7 @@ from sqlalchemy import select
from app.db import session from app.db import session
from app.models import Deployment, Secret, Setting, Tenant from app.models import Deployment, Secret, Setting, Tenant
from app.services import contract, crypto, guard, onyva_login, store from app.services import contract, crypto, guard, netcheck, onyva_login, store
from app.services.mittwald import DeployError, Mittwald, mask_env from app.services.mittwald import DeployError, Mittwald, mask_env
PORT = "8276" PORT = "8276"
@@ -162,9 +162,10 @@ def _hex_tag(tag: str) -> bool:
class HealthTimeout(DeployError): class HealthTimeout(DeployError):
"""Health-Check nicht grün geworden. answered=False: es kam nie eine gültige Antwort (Zertifikat/DNS/Container noch nicht bereit).""" """Health-Check nicht grün geworden. answered=False: es kam nie eine gültige Antwort (Zertifikat/DNS/Container noch nicht bereit)."""
def __init__(self, msg: str, answered: bool): def __init__(self, msg: str, answered: bool, dns: bool = False):
super().__init__(msg) super().__init__(msg)
self.answered = answered self.answered = answered
self.dns = dns # Hostname war vom Admin-Server aus nicht auflösbar (Resolver-Cache/DNS), nicht Fehler des Containers
def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, log, *, get=None, tries: int = 18, delay: float = 5.0, def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, log, *, get=None, tries: int = 18, delay: float = 5.0,
@@ -172,12 +173,8 @@ def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None,
"""Prüft nach dem Deploy, ob wirklich läuft, was wir ausgerollt haben (Vertrag: /health.core / .bundle). """Prüft nach dem Deploy, ob wirklich läuft, was wir ausgerollt haben (Vertrag: /health.core / .bundle).
Wirft DeployError, wenn der Container nach `tries` Versuchen nicht gesund ist oder die falsche Version meldet.""" Wirft DeployError, wenn der Container nach `tries` Versuchen nicht gesund ist oder die falsche Version meldet."""
if get is None: if get is None:
import httpx get = netcheck.get_json
last, answered, dns = "keine Antwort", False, False
def get(url):
r = httpx.get(url, timeout=8, follow_redirects=True)
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
last, answered = "keine Antwort", False
for i in range(tries): for i in range(tries):
try: try:
status, body = get(f"https://{host}/health") status, body = get(f"https://{host}/health")
@@ -197,18 +194,21 @@ def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None,
problems.append(f"Bundle {b.get('source')} v{b.get('version')}{' (veraltet)' if b.get('stale') else ''} statt admin v{expect_bundle}" problems.append(f"Bundle {b.get('source')} v{b.get('version')}{' (veraltet)' if b.get('stale') else ''} statt admin v{expect_bundle}"
+ (f" – {b.get('error')}" if b.get("error") else "")) + (f" – {b.get('error')}" if b.get("error") else ""))
if not problems: if not problems:
if body.get("_dns_fallback"):
log("(Hinweis: Hostname vom Admin-Server nicht auflösbar – Prüfung über DNS-over-HTTPS direkt zur IP)")
log(f"Health-Check https://{host}/health ✔" + (f" · Core {core['version'][:10]}" if core else "") log(f"Health-Check https://{host}/health ✔" + (f" · Core {core['version'][:10]}" if core else "")
+ (f" · Bundle v{body['bundle']['version']}" if body.get("bundle", {}).get("source") == "admin" else "")) + (f" · Bundle v{body['bundle']['version']}" if body.get("bundle", {}).get("source") == "admin" else ""))
return return
last = "; ".join(problems) last = "; ".join(problems)
except Exception as exc: # noqa: BLE001 except Exception as exc: # noqa: BLE001
last = f"{type(exc).__name__}: {exc}" last = f"{type(exc).__name__}: {exc}"
dns = netcheck.is_dns_failure_text(last)
if i < tries - 1: if i < tries - 1:
sleep(delay) sleep(delay)
raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered) raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered, dns and not answered)
def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None) -> bool: def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None, strict: bool = False) -> bool:
"""Führt einen Deployment-Datensatz aus. kind: dry-run | deploy | offboard-dry | offboard. """Führt einen Deployment-Datensatz aus. kind: dry-run | deploy | offboard-dry | offboard.
core_tag überschreibt den Core-Tag (Rollout/Rollback); `health(host, expect_sha, expect_bundle, log)` ist in Tests ersetzbar.""" core_tag überschreibt den Core-Tag (Rollout/Rollback); `health(host, expect_sha, expect_bundle, log)` ist in Tests ersetzbar."""
log = lambda m: _append(dep_id, m) # noqa: E731 log = lambda m: _append(dep_id, m) # noqa: E731
@@ -276,11 +276,16 @@ def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None,
args = (host, tag, ver if source == "admin" else None, log) args = (host, tag, ver if source == "admin" else None, log)
health(*args) if health else verify_health(*args, **kw) health(*args) if health else verify_health(*args, **kw)
except HealthTimeout as exc: except HealthTimeout as exc:
if fresh and not exc.answered: # nur „noch nicht erreichbar“ bei neuem Ingress: Deploy ist durch, Prüfung später wiederholen # Warnung statt Fehler: neuer Ingress ohne Antwort (Zertifikat/DNS) – oder (außerhalb strikter Rollouts) ein Namensauflösungsproblem des Admin-Servers
if not exc.answered and (fresh or (exc.dns and not strict)):
warning = True warning = True
log(f"⚠ {exc}") log(f"⚠ {exc}")
log("⚠ Container und Ingress sind angelegt, aber die Adresse antwortet noch nicht (Zertifikat/DNS?). " if exc.dns:
"Später „Health erneut prüfen“ (Deploy-Tab) oder „Läuft?“ (Übersicht).") log("⚠ Der Admin-Server kann den Hostnamen nicht auflösen (Resolver-Cache/DNS) – der Deploy selbst ist geschrieben. "
"Prüfe im Browser, ob die Adresse läuft, und später „Health erneut prüfen“.")
else:
log("⚠ Container und Ingress sind angelegt, aber die Adresse antwortet noch nicht (Zertifikat/DNS?). "
"Später „Health erneut prüfen“ (Deploy-Tab) oder „Läuft?“ (Übersicht).")
else: else:
raise raise
log("✔ Fertig" if live and not warning else ("Fertig mit Warnung" if warning else "Dry-Run beendet – es wurde nichts geschrieben.")) log("✔ Fertig" if live and not warning else ("Fertig mit Warnung" if warning else "Dry-Run beendet – es wurde nichts geschrieben."))
@@ -347,7 +352,7 @@ def run_sequence(tenant_ids: list[str], kind: str, actor: str, *, batch: str, co
continue continue
with session() as db: with session() as db:
db.get(Deployment, did).status = "running" db.get(Deployment, did).status = "running"
ok = execute(did, tid, kind, http, oidc_client, core_tag, health) ok = execute(did, tid, kind, http, oidc_client, core_tag, health, strict=bool(core_tag))
_finish(did, "warning" if ok == "warning" else ("success" if ok else "failed")) _finish(did, "warning" if ok == "warning" else ("success" if ok else "failed"))
results[tid] = "warning" if ok == "warning" else ("success" if ok else "failed") results[tid] = "warning" if ok == "warning" else ("success" if ok else "failed")
failed = not ok failed = not ok

60
app/services/netcheck.py Normal file
View File

@@ -0,0 +1,60 @@
"""HTTP-GET mit DNS-Rückfall für Health-Checks.
Der Resolver des Admin-Servers (Coolify/Docker) cacht „Name nicht gefunden“ für neue Hostnamen oft lange – obwohl der Name öffentlich
längst existiert. Schlägt die Auflösung fehl, fragen wir die Adresse per DNS-over-HTTPS (Cloudflare, dann Google) ab und verbinden uns
direkt zur IP (Host-Header + SNI = Hostname, Zertifikat wird normal geprüft).
"""
from __future__ import annotations
import socket
from urllib.parse import urlsplit
import httpx
def _is_dns_error(exc: BaseException) -> bool:
seen = set()
while exc is not None and id(exc) not in seen:
seen.add(id(exc))
if isinstance(exc, socket.gaierror):
return True
exc = exc.__cause__ or exc.__context__
return False
def is_dns_failure_text(msg: str) -> bool:
m = msg.lower()
return "name or service not known" in m or "temporary failure in name resolution" in m or "nodename nor servname" in m or "gaierror" in m
def doh_resolve(host: str, timeout: float = 5.0) -> list[str]:
for url in ("https://cloudflare-dns.com/dns-query", "https://dns.google/resolve"):
try:
r = httpx.get(url, params={"name": host, "type": "A"}, headers={"accept": "application/dns-json"}, timeout=timeout)
ips = [a["data"] for a in r.json().get("Answer", []) if a.get("type") == 1]
if ips:
return ips
except Exception: # noqa: BLE001
continue
return []
def get_json(url: str, timeout: float = 8.0) -> tuple[int, dict]:
"""GET mit DNS-Rückfall → (status, json-dict oder {}). Bei Rückfall enthält das Dict `_dns_fallback: true`."""
def parse(r):
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
try:
return parse(httpx.get(url, timeout=timeout, follow_redirects=True))
except httpx.ConnectError as exc:
if not _is_dns_error(exc):
raise
parts = urlsplit(url)
for ip in doh_resolve(parts.hostname or ""):
try:
r = httpx.get(f"{parts.scheme}://{ip}{parts.path or '/'}", headers={"Host": parts.hostname}, extensions={"sni_hostname": parts.hostname},
timeout=timeout, follow_redirects=True)
status, body = parse(r)
return status, {**body, "_dns_fallback": True}
except Exception: # noqa: BLE001
continue
raise

View File

@@ -15,9 +15,8 @@ TTL = 30.0
def _http_get(url: str): def _http_get(url: str):
import httpx from app.services import netcheck
r = httpx.get(url, timeout=6, follow_redirects=True) return netcheck.get_json(url, timeout=6)
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
def probe(host: str, expect_tag: str, expect_bundle: int | None, get=_http_get) -> dict: def probe(host: str, expect_tag: str, expect_bundle: int | None, get=_http_get) -> dict:

View File

@@ -47,14 +47,18 @@ def create(db: Session, tenant_id: str, name: str, clone_from: str | None, actor
def secret_rows(db: Session, tenant_id: str) -> list[dict]: def secret_rows(db: Session, tenant_id: str) -> list[dict]:
from app.services import contract
t = db.get(Tenant, tenant_id)
cfg, _ = store.content(db, t, "current") if t else ({}, {})
required = [k for k in contract.required_secrets(cfg) if k not in ("CORTECS_API_KEY",)] # geteilte Secrets stehen unter Einstellungen
rows = {s.key: s for s in db.scalars(select(Secret).where(Secret.tenant_id == tenant_id))} rows = {s.key: s for s in db.scalars(select(Secret).where(Secret.tenant_id == tenant_id))}
keys = list(dict.fromkeys(defaults.REQUIRED_SECRETS + [k for k in rows if k not in defaults.REQUIRED_SECRETS])) keys = list(dict.fromkeys(required + [k for k in rows if k not in required]))
out = [] out = []
for k in keys: for k in keys:
r = rows.get(k) r = rows.get(k)
out.append({ out.append({
"key": k, "set": bool(r and r.value_enc), "source": r.source if r else "", "key": k, "set": bool(r and r.value_enc), "source": r.source if r else "",
"required": k in defaults.REQUIRED_SECRETS, "help": defaults.SECRET_HELP.get(k, ""), "required": k in required, "help": defaults.SECRET_HELP.get(k, ""),
"updated_at": r.updated_at.isoformat() if r and r.updated_at else None, "updated_at": r.updated_at.isoformat() if r and r.updated_at else None,
}) })
return out return out

View File

@@ -85,7 +85,8 @@ def test_required_secrets_follow_schema_conditions():
assert "MISTRAL_API_KEY" not in contract.required_secrets({"stt": {"provider": "cortecs"}}) assert "MISTRAL_API_KEY" not in contract.required_secrets({"stt": {"provider": "cortecs"}})
assert "MISTRAL_API_KEY" in contract.required_secrets({"stt": {"provider": "mistral"}}) assert "MISTRAL_API_KEY" in contract.required_secrets({"stt": {"provider": "mistral"}})
assert "MISTRAL_API_KEY" in contract.required_secrets({}) assert "MISTRAL_API_KEY" in contract.required_secrets({})
assert {"OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "COST_INGEST_KEY", "CORTECS_API_KEY"} <= set(contract.required_secrets({})) assert {"OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "CORTECS_API_KEY"} <= set(contract.required_secrets({}))
assert "COST_INGEST_KEY" not in contract.required_secrets({}) # optional: ohne Key bleibt die Kostenerfassung inaktiv
# ── Health-Verifikation & Zustandsabgleich ────────────────────────────────── # ── Health-Verifikation & Zustandsabgleich ──────────────────────────────────
@@ -560,3 +561,52 @@ def test_wrong_version_or_existing_ingress_still_fails(client, live_env):
never = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("keine Antwort", answered=False)) # noqa: E731 never = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("keine Antwort", answered=False)) # noqa: E731
d2 = client.get(f"/api/deployments/{deploy.start('hc2', 'deploy', 't', http=WithIngress(), health=never, wait=True)}").json() d2 = client.get(f"/api/deployments/{deploy.start('hc2', 'deploy', 't', http=WithIngress(), health=never, wait=True)}").json()
assert d2["status"] == "failed" assert d2["status"] == "failed"
# ── DNS-Rückfall beim Health-Check ──────────────────────────────────────────
def test_dns_failure_falls_back_to_doh_and_connects_by_ip(monkeypatch):
import socket
import httpx
from app.services import netcheck
calls = []
class Resp:
status_code, headers = 200, {"content-type": "application/json"}
def __init__(self, body): self._b = body
def json(self): return self._b
def fake_get(url, **kw):
calls.append((url, kw.get("headers"), kw.get("extensions")))
if url == "https://demo.example.de/health":
raise httpx.ConnectError("[Errno -2] Name or service not known") from socket.gaierror(-2, "Name or service not known")
if "dns-query" in url:
return Resp({"Answer": [{"type": 1, "data": "203.0.113.7"}]})
return Resp({"status": "ok", "core": {"version": "abcdef1234567"}})
monkeypatch.setattr(httpx, "get", fake_get)
status, body = netcheck.get_json("https://demo.example.de/health")
assert status == 200 and body["_dns_fallback"] and body["core"]["version"].startswith("abcdef1")
ip_call = calls[-1]
assert ip_call[0] == "https://203.0.113.7/health" and ip_call[1] == {"Host": "demo.example.de"} and ip_call[2] == {"sni_hostname": "demo.example.de"}
# Verbindungsfehler ohne DNS-Ursache werden NICHT umgangen
monkeypatch.setattr(httpx, "get", lambda url, **kw: (_ for _ in ()).throw(httpx.ConnectError("refused")))
with pytest.raises(httpx.ConnectError):
netcheck.get_json("https://demo.example.de/health")
def test_unresolvable_host_is_a_warning_for_single_deploys_but_fails_strict_rollouts(client, live_env):
from app.services.deploy import HealthTimeout
_ready(client, "dn1")
class WithIngress(FakeMW):
def __call__(self, method, url, headers, body):
if method == "GET" and "ingresses" in url:
return 200, [{"hostname": "dn1.onyscribe.de", "id": "i1"}], ""
return super().__call__(method, url, headers, body)
dns = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("Name or service not known", answered=False, dns=True)) # noqa: E731
d = client.get(f"/api/deployments/{deploy.start('dn1', 'deploy', 't', http=WithIngress(), health=dns, wait=True)}").json()
assert d["status"] == "warning" and "kann den Hostnamen nicht auflösen" in d["log"] # bestehender Ingress, aber nur DNS-Problem des Admin-Servers
# strikt (Core-Rollout): ohne neuen Ingress bleibt es ein Fehler
rid = _approved_release(client, "5" * 40)
batch, _ = releases.rollout(rid, "t", dry_run=False, http=WithIngress(), health=dns, wait=True)
assert [r["status"] for r in client.get(f"/api/deployments?batch={batch}").json() if r["tenant"] == "dn1"] == ["failed"]

View File

@@ -477,9 +477,8 @@ def test_deploy_runs_oidc_sync_before_service_patch(client, login, monkeypatch):
# Pflicht-Secrets laut Core-Schema: erst fehlen sie → Deploy wird VOR dem Service-Eingriff blockiert # Pflicht-Secrets laut Core-Schema: erst fehlen sie → Deploy wird VOR dem Service-Eingriff blockiert
blocked = deploy.start("klinik5", "deploy", "test", http=lambda *a: (200, {}, ""), oidc_client=None, wait=True, health=lambda *a: None) blocked = deploy.start("klinik5", "deploy", "test", http=lambda *a: (200, {}, ""), oidc_client=None, wait=True, health=lambda *a: None)
d0 = client.get(f"/api/deployments/{blocked}").json() d0 = client.get(f"/api/deployments/{blocked}").json()
assert d0["status"] == "failed" and ("COST_INGEST_KEY" in d0["log"] or "OIDC-Abgleich" in d0["log"] or "ONYVA_LOGIN_API_KEY" in d0["log"]) assert d0["status"] == "failed" and "Pflicht-Secrets ohne Wert: CORTECS_API_KEY" in d0["log"]
client.put("/api/shared-secrets/CORTECS_API_KEY", json={"value": "ck"}) client.put("/api/shared-secrets/CORTECS_API_KEY", json={"value": "ck"})
client.put("/api/tenants/klinik5/secrets/COST_INGEST_KEY", json={"value": "ik"})
order = [] order = []
stack = {"services": {"legacy": {"id": "svc-0"}, "app-klinik5": {"id": "svc-5"}}} stack = {"services": {"legacy": {"id": "svc-0"}, "app-klinik5": {"id": "svc-5"}}}

View File

@@ -1 +1 @@
13055b811ca0030841f41ce1f10e4f30cba1a7db admin-platform-integration d6e72060858f584661bafc6396d40319b11e5afc admin-platform-integration

View File

@@ -1 +1 @@
{"sha": "13055b811ca0030841f41ce1f10e4f30cba1a7db", "built_at": "2026-10-15T13:26:28Z"} {"sha": "d6e72060858f584661bafc6396d40319b11e5afc", "built_at": "2026-10-15T14:36:00Z"}

View File

@@ -813,12 +813,11 @@
"name": "COST_INGEST_KEY", "name": "COST_INGEST_KEY",
"kind": "secret", "kind": "secret",
"scope": "tenant", "scope": "tenant",
"label": "Cost-Ingest-Key", "label": "Cost-Ingest-Key (optional)",
"help": "Schlüssel für die Kostenerfassung.", "help": "Optional – ohne Key bleibt die Kostenerfassung inaktiv (core/cost.py).",
"default": null, "default": null,
"group": "Secrets", "group": "Secrets",
"setting": "cost_ingest_key", "setting": "cost_ingest_key"
"required": true
}, },
{ {
"name": "TENANT", "name": "TENANT",