Ruben/Onyscribe-Admin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Health-Check: DNS-over-HTTPS-Rückfall bei nicht auflösbarem Hostnamen, DNS-Problem des Admin-Servers nur Warnung (außer strikte Rollouts); COST_INGEST_KEY optional (Schema-getrieben)

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
RubenRWU
2026-10-15 16:36:54 +02:00
parent 31a99e31c0
commit 9c352be05b
10 changed files with 146 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/services/defaults.py
View File

@@ -17,13 +17,13 @@ COLOR_LABELS = {
}
# Secret-Schlüssel = Env-Namen im Container (siehe infra/deploy.py)
REQUIRED_SECRETS = ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "COST_INGEST_KEY"]
REQUIRED_SECRETS = ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET"] # Rückfall; maßgeblich ist das Schema des Cores (contract.required_secrets)
OPTIONAL_SECRETS = ["MISTRAL_API_KEY"]
SECRET_HELP = {
"OIDC_CLIENT_ID": "Client-ID aus login.onyva.dev (manuell angelegt)",
"OIDC_CLIENT_SECRET": "Client-Secret aus login.onyva.dev",
"SESSION_SECRET": "Signiert Session-Cookies – kann hier erzeugt werden",
"COST_INGEST_KEY": "Schlüssel für die Kostenerfassung (Cost-API)",
"COST_INGEST_KEY": "Optional – Schlüssel für die Kostenerfassung (Cost-API); ohne Key bleibt sie inaktiv",
"MISTRAL_API_KEY": "Nur nötig bei STT-Provider „mistral“ (bei „cortecs“ wird der geteilte Key genutzt)",
"ADMIN_BUNDLE_TOKEN": "Token, mit dem der Container sein Bundle bei der Admin-Plattform abholt",
}

29
app/services/deploy.py
View File

@@ -16,7 +16,7 @@ from sqlalchemy import select
from app.db import session
from app.models import Deployment, Secret, Setting, Tenant
from app.services import contract, crypto, guard, onyva_login, store
from app.services import contract, crypto, guard, netcheck, onyva_login, store
from app.services.mittwald import DeployError, Mittwald, mask_env
PORT = "8276"
@@ -162,9 +162,10 @@ def _hex_tag(tag: str) -> bool:
class HealthTimeout(DeployError):
"""Health-Check nicht grün geworden. answered=False: es kam nie eine gültige Antwort (Zertifikat/DNS/Container noch nicht bereit)."""
def __init__(self, msg: str, answered: bool):
def __init__(self, msg: str, answered: bool, dns: bool = False):
super().__init__(msg)
self.answered = answered
self.dns = dns # Hostname war vom Admin-Server aus nicht auflösbar (Resolver-Cache/DNS), nicht Fehler des Containers
def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, log, *, get=None, tries: int = 18, delay: float = 5.0,
@@ -172,12 +173,8 @@ def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None,
"""Prüft nach dem Deploy, ob wirklich läuft, was wir ausgerollt haben (Vertrag: /health.core / .bundle).
Wirft DeployError, wenn der Container nach `tries` Versuchen nicht gesund ist oder die falsche Version meldet."""
if get is None:
import httpx
def get(url):
r = httpx.get(url, timeout=8, follow_redirects=True)
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
last, answered = "keine Antwort", False
get = netcheck.get_json
last, answered, dns = "keine Antwort", False, False
for i in range(tries):
try:
status, body = get(f"https://{host}/health")
@@ -197,18 +194,21 @@ def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None,
problems.append(f"Bundle {b.get('source')} v{b.get('version')}{' (veraltet)' if b.get('stale') else ''} statt admin v{expect_bundle}"
+ (f" – {b.get('error')}" if b.get("error") else ""))
if not problems:
if body.get("_dns_fallback"):
log("(Hinweis: Hostname vom Admin-Server nicht auflösbar – Prüfung über DNS-over-HTTPS direkt zur IP)")
log(f"Health-Check https://{host}/health ✔" + (f" · Core {core['version'][:10]}" if core else "")
+ (f" · Bundle v{body['bundle']['version']}" if body.get("bundle", {}).get("source") == "admin" else ""))
return
last = "; ".join(problems)
except Exception as exc: # noqa: BLE001
last = f"{type(exc).__name__}: {exc}"
dns = netcheck.is_dns_failure_text(last)
if i < tries - 1:
sleep(delay)
raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered)
raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered, dns and not answered)
def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None) -> bool:
def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None, strict: bool = False) -> bool:
"""Führt einen Deployment-Datensatz aus. kind: dry-run | deploy | offboard-dry | offboard.
core_tag überschreibt den Core-Tag (Rollout/Rollback); `health(host, expect_sha, expect_bundle, log)` ist in Tests ersetzbar."""
log = lambda m: _append(dep_id, m) # noqa: E731
@@ -276,9 +276,14 @@ def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None,
args = (host, tag, ver if source == "admin" else None, log)
health(*args) if health else verify_health(*args, **kw)
except HealthTimeout as exc:
if fresh and not exc.answered: # nur „noch nicht erreichbar“ bei neuem Ingress: Deploy ist durch, Prüfung später wiederholen
# Warnung statt Fehler: neuer Ingress ohne Antwort (Zertifikat/DNS) – oder (außerhalb strikter Rollouts) ein Namensauflösungsproblem des Admin-Servers
if not exc.answered and (fresh or (exc.dns and not strict)):
warning = True
log(f"⚠ {exc}")
if exc.dns:
log("⚠ Der Admin-Server kann den Hostnamen nicht auflösen (Resolver-Cache/DNS) – der Deploy selbst ist geschrieben. "
"Prüfe im Browser, ob die Adresse läuft, und später „Health erneut prüfen“.")
else:
log("⚠ Container und Ingress sind angelegt, aber die Adresse antwortet noch nicht (Zertifikat/DNS?). "
"Später „Health erneut prüfen“ (Deploy-Tab) oder „Läuft?“ (Übersicht).")
else:
@@ -347,7 +352,7 @@ def run_sequence(tenant_ids: list[str], kind: str, actor: str, *, batch: str, co
continue
with session() as db:
db.get(Deployment, did).status = "running"
ok = execute(did, tid, kind, http, oidc_client, core_tag, health)
ok = execute(did, tid, kind, http, oidc_client, core_tag, health, strict=bool(core_tag))
_finish(did, "warning" if ok == "warning" else ("success" if ok else "failed"))
results[tid] = "warning" if ok == "warning" else ("success" if ok else "failed")
failed = not ok

60
app/services/netcheck.py Normal file
View File

@@ -0,0 +1,60 @@
"""HTTP-GET mit DNS-Rückfall für Health-Checks.
Der Resolver des Admin-Servers (Coolify/Docker) cacht „Name nicht gefunden“ für neue Hostnamen oft lange – obwohl der Name öffentlich
längst existiert. Schlägt die Auflösung fehl, fragen wir die Adresse per DNS-over-HTTPS (Cloudflare, dann Google) ab und verbinden uns
direkt zur IP (Host-Header + SNI = Hostname, Zertifikat wird normal geprüft).
"""
from __future__ import annotations
import socket
from urllib.parse import urlsplit
import httpx
def _is_dns_error(exc: BaseException) -> bool:
seen = set()
while exc is not None and id(exc) not in seen:
seen.add(id(exc))
if isinstance(exc, socket.gaierror):
return True
exc = exc.__cause__ or exc.__context__
return False
def is_dns_failure_text(msg: str) -> bool:
m = msg.lower()
return "name or service not known" in m or "temporary failure in name resolution" in m or "nodename nor servname" in m or "gaierror" in m
def doh_resolve(host: str, timeout: float = 5.0) -> list[str]:
for url in ("https://cloudflare-dns.com/dns-query", "https://dns.google/resolve"):
try:
r = httpx.get(url, params={"name": host, "type": "A"}, headers={"accept": "application/dns-json"}, timeout=timeout)
ips = [a["data"] for a in r.json().get("Answer", []) if a.get("type") == 1]
if ips:
return ips
except Exception: # noqa: BLE001
continue
return []
def get_json(url: str, timeout: float = 8.0) -> tuple[int, dict]:
"""GET mit DNS-Rückfall → (status, json-dict oder {}). Bei Rückfall enthält das Dict `_dns_fallback: true`."""
def parse(r):
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
try:
return parse(httpx.get(url, timeout=timeout, follow_redirects=True))
except httpx.ConnectError as exc:
if not _is_dns_error(exc):
raise
parts = urlsplit(url)
for ip in doh_resolve(parts.hostname or ""):
try:
r = httpx.get(f"{parts.scheme}://{ip}{parts.path or '/'}", headers={"Host": parts.hostname}, extensions={"sni_hostname": parts.hostname},
timeout=timeout, follow_redirects=True)
status, body = parse(r)
return status, {**body, "_dns_fallback": True}
except Exception: # noqa: BLE001
continue
raise

5
app/services/status.py
View File

@@ -15,9 +15,8 @@ TTL = 30.0
def _http_get(url: str):
import httpx
r = httpx.get(url, timeout=6, follow_redirects=True)
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
from app.services import netcheck
return netcheck.get_json(url, timeout=6)
def probe(host: str, expect_tag: str, expect_bundle: int | None, get=_http_get) -> dict:

8
app/services/tenants.py
View File

@@ -47,14 +47,18 @@ def create(db: Session, tenant_id: str, name: str, clone_from: str | None, actor
def secret_rows(db: Session, tenant_id: str) -> list[dict]:
from app.services import contract
t = db.get(Tenant, tenant_id)
cfg, _ = store.content(db, t, "current") if t else ({}, {})
required = [k for k in contract.required_secrets(cfg) if k not in ("CORTECS_API_KEY",)] # geteilte Secrets stehen unter Einstellungen
rows = {s.key: s for s in db.scalars(select(Secret).where(Secret.tenant_id == tenant_id))}
keys = list(dict.fromkeys(defaults.REQUIRED_SECRETS + [k for k in rows if k not in defaults.REQUIRED_SECRETS]))
keys = list(dict.fromkeys(required + [k for k in rows if k not in required]))
out = []
for k in keys:
r = rows.get(k)
out.append({
"key": k, "set": bool(r and r.value_enc), "source": r.source if r else "",
"required": k in defaults.REQUIRED_SECRETS, "help": defaults.SECRET_HELP.get(k, ""),
"required": k in required, "help": defaults.SECRET_HELP.get(k, ""),
"updated_at": r.updated_at.isoformat() if r and r.updated_at else None,
})
return out

52
tests/test_core_contract.py
View File

@@ -85,7 +85,8 @@ def test_required_secrets_follow_schema_conditions():
assert "MISTRAL_API_KEY" not in contract.required_secrets({"stt": {"provider": "cortecs"}})
assert "MISTRAL_API_KEY" in contract.required_secrets({"stt": {"provider": "mistral"}})
assert "MISTRAL_API_KEY" in contract.required_secrets({})
assert {"OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "COST_INGEST_KEY", "CORTECS_API_KEY"} <= set(contract.required_secrets({}))
assert {"OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "CORTECS_API_KEY"} <= set(contract.required_secrets({}))
assert "COST_INGEST_KEY" not in contract.required_secrets({}) # optional: ohne Key bleibt die Kostenerfassung inaktiv
# ── Health-Verifikation & Zustandsabgleich ──────────────────────────────────
@@ -560,3 +561,52 @@ def test_wrong_version_or_existing_ingress_still_fails(client, live_env):
never = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("keine Antwort", answered=False)) # noqa: E731
d2 = client.get(f"/api/deployments/{deploy.start('hc2', 'deploy', 't', http=WithIngress(), health=never, wait=True)}").json()
assert d2["status"] == "failed"
# ── DNS-Rückfall beim Health-Check ──────────────────────────────────────────
def test_dns_failure_falls_back_to_doh_and_connects_by_ip(monkeypatch):
import socket
import httpx
from app.services import netcheck
calls = []
class Resp:
status_code, headers = 200, {"content-type": "application/json"}
def __init__(self, body): self._b = body
def json(self): return self._b
def fake_get(url, **kw):
calls.append((url, kw.get("headers"), kw.get("extensions")))
if url == "https://demo.example.de/health":
raise httpx.ConnectError("[Errno -2] Name or service not known") from socket.gaierror(-2, "Name or service not known")
if "dns-query" in url:
return Resp({"Answer": [{"type": 1, "data": "203.0.113.7"}]})
return Resp({"status": "ok", "core": {"version": "abcdef1234567"}})
monkeypatch.setattr(httpx, "get", fake_get)
status, body = netcheck.get_json("https://demo.example.de/health")
assert status == 200 and body["_dns_fallback"] and body["core"]["version"].startswith("abcdef1")
ip_call = calls[-1]
assert ip_call[0] == "https://203.0.113.7/health" and ip_call[1] == {"Host": "demo.example.de"} and ip_call[2] == {"sni_hostname": "demo.example.de"}
# Verbindungsfehler ohne DNS-Ursache werden NICHT umgangen
monkeypatch.setattr(httpx, "get", lambda url, **kw: (_ for _ in ()).throw(httpx.ConnectError("refused")))
with pytest.raises(httpx.ConnectError):
netcheck.get_json("https://demo.example.de/health")
def test_unresolvable_host_is_a_warning_for_single_deploys_but_fails_strict_rollouts(client, live_env):
from app.services.deploy import HealthTimeout
_ready(client, "dn1")
class WithIngress(FakeMW):
def __call__(self, method, url, headers, body):
if method == "GET" and "ingresses" in url:
return 200, [{"hostname": "dn1.onyscribe.de", "id": "i1"}], ""
return super().__call__(method, url, headers, body)
dns = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("Name or service not known", answered=False, dns=True)) # noqa: E731
d = client.get(f"/api/deployments/{deploy.start('dn1', 'deploy', 't', http=WithIngress(), health=dns, wait=True)}").json()
assert d["status"] == "warning" and "kann den Hostnamen nicht auflösen" in d["log"] # bestehender Ingress, aber nur DNS-Problem des Admin-Servers
# strikt (Core-Rollout): ohne neuen Ingress bleibt es ein Fehler
rid = _approved_release(client, "5" * 40)
batch, _ = releases.rollout(rid, "t", dry_run=False, http=WithIngress(), health=dns, wait=True)
assert [r["status"] for r in client.get(f"/api/deployments?batch={batch}").json() if r["tenant"] == "dn1"] == ["failed"]

3
tests/test_flow.py
View File

@@ -477,9 +477,8 @@ def test_deploy_runs_oidc_sync_before_service_patch(client, login, monkeypatch):
# Pflicht-Secrets laut Core-Schema: erst fehlen sie → Deploy wird VOR dem Service-Eingriff blockiert
blocked = deploy.start("klinik5", "deploy", "test", http=lambda *a: (200, {}, ""), oidc_client=None, wait=True, health=lambda *a: None)
d0 = client.get(f"/api/deployments/{blocked}").json()
assert d0["status"] == "failed" and ("COST_INGEST_KEY" in d0["log"] or "OIDC-Abgleich" in d0["log"] or "ONYVA_LOGIN_API_KEY" in d0["log"])
assert d0["status"] == "failed" and "Pflicht-Secrets ohne Wert: CORTECS_API_KEY" in d0["log"]
client.put("/api/shared-secrets/CORTECS_API_KEY", json={"value": "ck"})
client.put("/api/tenants/klinik5/secrets/COST_INGEST_KEY", json={"value": "ik"})
order = []
stack = {"services": {"legacy": {"id": "svc-0"}, "app-klinik5": {"id": "svc-5"}}}

2
vendor/platform/CORE_VERSION vendored
View File

@@ -1 +1 @@
13055b811ca0030841f41ce1f10e4f30cba1a7db admin-platform-integration
d6e72060858f584661bafc6396d40319b11e5afc admin-platform-integration

2
vendor/platform/core/BUILD_INFO.json vendored
View File

@@ -1 +1 @@
{"sha": "13055b811ca0030841f41ce1f10e4f30cba1a7db", "built_at": "2026-10-15T13:26:28Z"}
{"sha": "d6e72060858f584661bafc6396d40319b11e5afc", "built_at": "2026-10-15T14:36:00Z"}

7
vendor/platform/core/schema/tenant_schema.json vendored
View File

@@ -813,12 +813,11 @@
"name": "COST_INGEST_KEY",
"kind": "secret",
"scope": "tenant",
"label": "Cost-Ingest-Key",
"help": "Schlüssel für die Kostenerfassung.",
"label": "Cost-Ingest-Key (optional)",
"help": "Optional – ohne Key bleibt die Kostenerfassung inaktiv (core/cost.py).",
"default": null,
"group": "Secrets",
"setting": "cost_ingest_key",
"required": true
"setting": "cost_ingest_key"
},
{
"name": "TENANT",