From 9c352be05bc7713a5582be9658221dd2155c3326 Mon Sep 17 00:00:00 2001 From: RubenRWU Date: Thu, 15 Oct 2026 16:36:54 +0200 Subject: [PATCH] =?UTF-8?q?Health-Check:=20DNS-over-HTTPS-R=C3=BCckfall=20?= =?UTF-8?q?bei=20nicht=20aufl=C3=B6sbarem=20Hostnamen,=20DNS-Problem=20des?= =?UTF-8?q?=20Admin-Servers=20nur=20Warnung=20(au=C3=9Fer=20strikte=20Roll?= =?UTF-8?q?outs);=20COST=5FINGEST=5FKEY=20optional=20(Schema-getrieben)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- app/services/defaults.py | 4 +- app/services/deploy.py | 33 +++++----- app/services/netcheck.py | 60 +++++++++++++++++++ app/services/status.py | 5 +- app/services/tenants.py | 8 ++- tests/test_core_contract.py | 52 +++++++++++++++- tests/test_flow.py | 3 +- vendor/platform/CORE_VERSION | 2 +- vendor/platform/core/BUILD_INFO.json | 2 +- .../platform/core/schema/tenant_schema.json | 7 +-- 10 files changed, 146 insertions(+), 30 deletions(-) create mode 100644 app/services/netcheck.py diff --git a/app/services/defaults.py b/app/services/defaults.py index 7ff894d..f72f416 100644 --- a/app/services/defaults.py +++ b/app/services/defaults.py @@ -17,13 +17,13 @@ COLOR_LABELS = { } # Secret-Schlüssel = Env-Namen im Container (siehe infra/deploy.py) -REQUIRED_SECRETS = ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "COST_INGEST_KEY"] +REQUIRED_SECRETS = ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET"] # Rückfall; maßgeblich ist das Schema des Cores (contract.required_secrets) OPTIONAL_SECRETS = ["MISTRAL_API_KEY"] SECRET_HELP = { "OIDC_CLIENT_ID": "Client-ID aus login.onyva.dev (manuell angelegt)", "OIDC_CLIENT_SECRET": "Client-Secret aus login.onyva.dev", "SESSION_SECRET": "Signiert Session-Cookies – kann hier erzeugt werden", - "COST_INGEST_KEY": "Schlüssel für die Kostenerfassung (Cost-API)", + "COST_INGEST_KEY": "Optional – Schlüssel für die Kostenerfassung (Cost-API); ohne Key bleibt sie inaktiv", "MISTRAL_API_KEY": "Nur nötig bei STT-Provider „mistral“ (bei „cortecs“ wird der geteilte Key genutzt)", "ADMIN_BUNDLE_TOKEN": "Token, mit dem der Container sein Bundle bei der Admin-Plattform abholt", } diff --git a/app/services/deploy.py b/app/services/deploy.py index f32831f..e3eea47 100644 --- a/app/services/deploy.py +++ b/app/services/deploy.py @@ -16,7 +16,7 @@ from sqlalchemy import select from app.db import session from app.models import Deployment, Secret, Setting, Tenant -from app.services import contract, crypto, guard, onyva_login, store +from app.services import contract, crypto, guard, netcheck, onyva_login, store from app.services.mittwald import DeployError, Mittwald, mask_env PORT = "8276" @@ -162,9 +162,10 @@ def _hex_tag(tag: str) -> bool: class HealthTimeout(DeployError): """Health-Check nicht grün geworden. answered=False: es kam nie eine gültige Antwort (Zertifikat/DNS/Container noch nicht bereit).""" - def __init__(self, msg: str, answered: bool): + def __init__(self, msg: str, answered: bool, dns: bool = False): super().__init__(msg) self.answered = answered + self.dns = dns # Hostname war vom Admin-Server aus nicht auflösbar (Resolver-Cache/DNS), nicht Fehler des Containers def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, log, *, get=None, tries: int = 18, delay: float = 5.0, @@ -172,12 +173,8 @@ def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, """Prüft nach dem Deploy, ob wirklich läuft, was wir ausgerollt haben (Vertrag: /health.core / .bundle). Wirft DeployError, wenn der Container nach `tries` Versuchen nicht gesund ist oder die falsche Version meldet.""" if get is None: - import httpx - - def get(url): - r = httpx.get(url, timeout=8, follow_redirects=True) - return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {}) - last, answered = "keine Antwort", False + get = netcheck.get_json + last, answered, dns = "keine Antwort", False, False for i in range(tries): try: status, body = get(f"https://{host}/health") @@ -197,18 +194,21 @@ def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, problems.append(f"Bundle {b.get('source')} v{b.get('version')}{' (veraltet)' if b.get('stale') else ''} statt admin v{expect_bundle}" + (f" – {b.get('error')}" if b.get("error") else "")) if not problems: + if body.get("_dns_fallback"): + log("(Hinweis: Hostname vom Admin-Server nicht auflösbar – Prüfung über DNS-over-HTTPS direkt zur IP)") log(f"Health-Check https://{host}/health ✔" + (f" · Core {core['version'][:10]}" if core else "") + (f" · Bundle v{body['bundle']['version']}" if body.get("bundle", {}).get("source") == "admin" else "")) return last = "; ".join(problems) except Exception as exc: # noqa: BLE001 last = f"{type(exc).__name__}: {exc}" + dns = netcheck.is_dns_failure_text(last) if i < tries - 1: sleep(delay) - raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered) + raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered, dns and not answered) -def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None) -> bool: +def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None, strict: bool = False) -> bool: """Führt einen Deployment-Datensatz aus. kind: dry-run | deploy | offboard-dry | offboard. core_tag überschreibt den Core-Tag (Rollout/Rollback); `health(host, expect_sha, expect_bundle, log)` ist in Tests ersetzbar.""" log = lambda m: _append(dep_id, m) # noqa: E731 @@ -276,11 +276,16 @@ def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, args = (host, tag, ver if source == "admin" else None, log) health(*args) if health else verify_health(*args, **kw) except HealthTimeout as exc: - if fresh and not exc.answered: # nur „noch nicht erreichbar“ bei neuem Ingress: Deploy ist durch, Prüfung später wiederholen + # Warnung statt Fehler: neuer Ingress ohne Antwort (Zertifikat/DNS) – oder (außerhalb strikter Rollouts) ein Namensauflösungsproblem des Admin-Servers + if not exc.answered and (fresh or (exc.dns and not strict)): warning = True log(f"⚠ {exc}") - log("⚠ Container und Ingress sind angelegt, aber die Adresse antwortet noch nicht (Zertifikat/DNS?). " - "Später „Health erneut prüfen“ (Deploy-Tab) oder „Läuft?“ (Übersicht).") + if exc.dns: + log("⚠ Der Admin-Server kann den Hostnamen nicht auflösen (Resolver-Cache/DNS) – der Deploy selbst ist geschrieben. " + "Prüfe im Browser, ob die Adresse läuft, und später „Health erneut prüfen“.") + else: + log("⚠ Container und Ingress sind angelegt, aber die Adresse antwortet noch nicht (Zertifikat/DNS?). " + "Später „Health erneut prüfen“ (Deploy-Tab) oder „Läuft?“ (Übersicht).") else: raise log("✔ Fertig" if live and not warning else ("Fertig mit Warnung" if warning else "Dry-Run beendet – es wurde nichts geschrieben.")) @@ -347,7 +352,7 @@ def run_sequence(tenant_ids: list[str], kind: str, actor: str, *, batch: str, co continue with session() as db: db.get(Deployment, did).status = "running" - ok = execute(did, tid, kind, http, oidc_client, core_tag, health) + ok = execute(did, tid, kind, http, oidc_client, core_tag, health, strict=bool(core_tag)) _finish(did, "warning" if ok == "warning" else ("success" if ok else "failed")) results[tid] = "warning" if ok == "warning" else ("success" if ok else "failed") failed = not ok diff --git a/app/services/netcheck.py b/app/services/netcheck.py new file mode 100644 index 0000000..9ea95ce --- /dev/null +++ b/app/services/netcheck.py @@ -0,0 +1,60 @@ +"""HTTP-GET mit DNS-Rückfall für Health-Checks. + +Der Resolver des Admin-Servers (Coolify/Docker) cacht „Name nicht gefunden“ für neue Hostnamen oft lange – obwohl der Name öffentlich +längst existiert. Schlägt die Auflösung fehl, fragen wir die Adresse per DNS-over-HTTPS (Cloudflare, dann Google) ab und verbinden uns +direkt zur IP (Host-Header + SNI = Hostname, Zertifikat wird normal geprüft). +""" +from __future__ import annotations + +import socket +from urllib.parse import urlsplit + +import httpx + + +def _is_dns_error(exc: BaseException) -> bool: + seen = set() + while exc is not None and id(exc) not in seen: + seen.add(id(exc)) + if isinstance(exc, socket.gaierror): + return True + exc = exc.__cause__ or exc.__context__ + return False + + +def is_dns_failure_text(msg: str) -> bool: + m = msg.lower() + return "name or service not known" in m or "temporary failure in name resolution" in m or "nodename nor servname" in m or "gaierror" in m + + +def doh_resolve(host: str, timeout: float = 5.0) -> list[str]: + for url in ("https://cloudflare-dns.com/dns-query", "https://dns.google/resolve"): + try: + r = httpx.get(url, params={"name": host, "type": "A"}, headers={"accept": "application/dns-json"}, timeout=timeout) + ips = [a["data"] for a in r.json().get("Answer", []) if a.get("type") == 1] + if ips: + return ips + except Exception: # noqa: BLE001 + continue + return [] + + +def get_json(url: str, timeout: float = 8.0) -> tuple[int, dict]: + """GET mit DNS-Rückfall → (status, json-dict oder {}). Bei Rückfall enthält das Dict `_dns_fallback: true`.""" + def parse(r): + return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {}) + try: + return parse(httpx.get(url, timeout=timeout, follow_redirects=True)) + except httpx.ConnectError as exc: + if not _is_dns_error(exc): + raise + parts = urlsplit(url) + for ip in doh_resolve(parts.hostname or ""): + try: + r = httpx.get(f"{parts.scheme}://{ip}{parts.path or '/'}", headers={"Host": parts.hostname}, extensions={"sni_hostname": parts.hostname}, + timeout=timeout, follow_redirects=True) + status, body = parse(r) + return status, {**body, "_dns_fallback": True} + except Exception: # noqa: BLE001 + continue + raise diff --git a/app/services/status.py b/app/services/status.py index ab6248e..37f41c9 100644 --- a/app/services/status.py +++ b/app/services/status.py @@ -15,9 +15,8 @@ TTL = 30.0 def _http_get(url: str): - import httpx - r = httpx.get(url, timeout=6, follow_redirects=True) - return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {}) + from app.services import netcheck + return netcheck.get_json(url, timeout=6) def probe(host: str, expect_tag: str, expect_bundle: int | None, get=_http_get) -> dict: diff --git a/app/services/tenants.py b/app/services/tenants.py index c4dd6cc..60f662e 100644 --- a/app/services/tenants.py +++ b/app/services/tenants.py @@ -47,14 +47,18 @@ def create(db: Session, tenant_id: str, name: str, clone_from: str | None, actor def secret_rows(db: Session, tenant_id: str) -> list[dict]: + from app.services import contract + t = db.get(Tenant, tenant_id) + cfg, _ = store.content(db, t, "current") if t else ({}, {}) + required = [k for k in contract.required_secrets(cfg) if k not in ("CORTECS_API_KEY",)] # geteilte Secrets stehen unter Einstellungen rows = {s.key: s for s in db.scalars(select(Secret).where(Secret.tenant_id == tenant_id))} - keys = list(dict.fromkeys(defaults.REQUIRED_SECRETS + [k for k in rows if k not in defaults.REQUIRED_SECRETS])) + keys = list(dict.fromkeys(required + [k for k in rows if k not in required])) out = [] for k in keys: r = rows.get(k) out.append({ "key": k, "set": bool(r and r.value_enc), "source": r.source if r else "", - "required": k in defaults.REQUIRED_SECRETS, "help": defaults.SECRET_HELP.get(k, ""), + "required": k in required, "help": defaults.SECRET_HELP.get(k, ""), "updated_at": r.updated_at.isoformat() if r and r.updated_at else None, }) return out diff --git a/tests/test_core_contract.py b/tests/test_core_contract.py index 9b8af95..3bfc73e 100644 --- a/tests/test_core_contract.py +++ b/tests/test_core_contract.py @@ -85,7 +85,8 @@ def test_required_secrets_follow_schema_conditions(): assert "MISTRAL_API_KEY" not in contract.required_secrets({"stt": {"provider": "cortecs"}}) assert "MISTRAL_API_KEY" in contract.required_secrets({"stt": {"provider": "mistral"}}) assert "MISTRAL_API_KEY" in contract.required_secrets({}) - assert {"OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "COST_INGEST_KEY", "CORTECS_API_KEY"} <= set(contract.required_secrets({})) + assert {"OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "SESSION_SECRET", "CORTECS_API_KEY"} <= set(contract.required_secrets({})) + assert "COST_INGEST_KEY" not in contract.required_secrets({}) # optional: ohne Key bleibt die Kostenerfassung inaktiv # ── Health-Verifikation & Zustandsabgleich ────────────────────────────────── @@ -560,3 +561,52 @@ def test_wrong_version_or_existing_ingress_still_fails(client, live_env): never = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("keine Antwort", answered=False)) # noqa: E731 d2 = client.get(f"/api/deployments/{deploy.start('hc2', 'deploy', 't', http=WithIngress(), health=never, wait=True)}").json() assert d2["status"] == "failed" + + +# ── DNS-Rückfall beim Health-Check ────────────────────────────────────────── + +def test_dns_failure_falls_back_to_doh_and_connects_by_ip(monkeypatch): + import socket + import httpx + from app.services import netcheck + calls = [] + + class Resp: + status_code, headers = 200, {"content-type": "application/json"} + def __init__(self, body): self._b = body + def json(self): return self._b + + def fake_get(url, **kw): + calls.append((url, kw.get("headers"), kw.get("extensions"))) + if url == "https://demo.example.de/health": + raise httpx.ConnectError("[Errno -2] Name or service not known") from socket.gaierror(-2, "Name or service not known") + if "dns-query" in url: + return Resp({"Answer": [{"type": 1, "data": "203.0.113.7"}]}) + return Resp({"status": "ok", "core": {"version": "abcdef1234567"}}) + + monkeypatch.setattr(httpx, "get", fake_get) + status, body = netcheck.get_json("https://demo.example.de/health") + assert status == 200 and body["_dns_fallback"] and body["core"]["version"].startswith("abcdef1") + ip_call = calls[-1] + assert ip_call[0] == "https://203.0.113.7/health" and ip_call[1] == {"Host": "demo.example.de"} and ip_call[2] == {"sni_hostname": "demo.example.de"} + # Verbindungsfehler ohne DNS-Ursache werden NICHT umgangen + monkeypatch.setattr(httpx, "get", lambda url, **kw: (_ for _ in ()).throw(httpx.ConnectError("refused"))) + with pytest.raises(httpx.ConnectError): + netcheck.get_json("https://demo.example.de/health") + + +def test_unresolvable_host_is_a_warning_for_single_deploys_but_fails_strict_rollouts(client, live_env): + from app.services.deploy import HealthTimeout + _ready(client, "dn1") + class WithIngress(FakeMW): + def __call__(self, method, url, headers, body): + if method == "GET" and "ingresses" in url: + return 200, [{"hostname": "dn1.onyscribe.de", "id": "i1"}], "" + return super().__call__(method, url, headers, body) + dns = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("Name or service not known", answered=False, dns=True)) # noqa: E731 + d = client.get(f"/api/deployments/{deploy.start('dn1', 'deploy', 't', http=WithIngress(), health=dns, wait=True)}").json() + assert d["status"] == "warning" and "kann den Hostnamen nicht auflösen" in d["log"] # bestehender Ingress, aber nur DNS-Problem des Admin-Servers + # strikt (Core-Rollout): ohne neuen Ingress bleibt es ein Fehler + rid = _approved_release(client, "5" * 40) + batch, _ = releases.rollout(rid, "t", dry_run=False, http=WithIngress(), health=dns, wait=True) + assert [r["status"] for r in client.get(f"/api/deployments?batch={batch}").json() if r["tenant"] == "dn1"] == ["failed"] diff --git a/tests/test_flow.py b/tests/test_flow.py index 8a7cf95..d392ff2 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -477,9 +477,8 @@ def test_deploy_runs_oidc_sync_before_service_patch(client, login, monkeypatch): # Pflicht-Secrets laut Core-Schema: erst fehlen sie → Deploy wird VOR dem Service-Eingriff blockiert blocked = deploy.start("klinik5", "deploy", "test", http=lambda *a: (200, {}, ""), oidc_client=None, wait=True, health=lambda *a: None) d0 = client.get(f"/api/deployments/{blocked}").json() - assert d0["status"] == "failed" and ("COST_INGEST_KEY" in d0["log"] or "OIDC-Abgleich" in d0["log"] or "ONYVA_LOGIN_API_KEY" in d0["log"]) + assert d0["status"] == "failed" and "Pflicht-Secrets ohne Wert: CORTECS_API_KEY" in d0["log"] client.put("/api/shared-secrets/CORTECS_API_KEY", json={"value": "ck"}) - client.put("/api/tenants/klinik5/secrets/COST_INGEST_KEY", json={"value": "ik"}) order = [] stack = {"services": {"legacy": {"id": "svc-0"}, "app-klinik5": {"id": "svc-5"}}} diff --git a/vendor/platform/CORE_VERSION b/vendor/platform/CORE_VERSION index ca68dcd..5f81a10 100644 --- a/vendor/platform/CORE_VERSION +++ b/vendor/platform/CORE_VERSION @@ -1 +1 @@ -13055b811ca0030841f41ce1f10e4f30cba1a7db admin-platform-integration +d6e72060858f584661bafc6396d40319b11e5afc admin-platform-integration diff --git a/vendor/platform/core/BUILD_INFO.json b/vendor/platform/core/BUILD_INFO.json index 65fb146..846c40c 100644 --- a/vendor/platform/core/BUILD_INFO.json +++ b/vendor/platform/core/BUILD_INFO.json @@ -1 +1 @@ -{"sha": "13055b811ca0030841f41ce1f10e4f30cba1a7db", "built_at": "2026-10-15T13:26:28Z"} +{"sha": "d6e72060858f584661bafc6396d40319b11e5afc", "built_at": "2026-10-15T14:36:00Z"} diff --git a/vendor/platform/core/schema/tenant_schema.json b/vendor/platform/core/schema/tenant_schema.json index 9fa17f7..b8b9a34 100644 --- a/vendor/platform/core/schema/tenant_schema.json +++ b/vendor/platform/core/schema/tenant_schema.json @@ -813,12 +813,11 @@ "name": "COST_INGEST_KEY", "kind": "secret", "scope": "tenant", - "label": "Cost-Ingest-Key", - "help": "Schlüssel für die Kostenerfassung.", + "label": "Cost-Ingest-Key (optional)", + "help": "Optional – ohne Key bleibt die Kostenerfassung inaktiv (core/cost.py).", "default": null, "group": "Secrets", - "setting": "cost_ingest_key", - "required": true + "setting": "cost_ingest_key" }, { "name": "TENANT",