diff --git a/app/routers/api.py b/app/routers/api.py index 6e991c6..b1bf2d7 100644 --- a/app/routers/api.py +++ b/app/routers/api.py @@ -474,6 +474,12 @@ def tenant_preflight(t: Tenant = Depends(tenant_or_404)): return preflight.run(t.id) +@router.post("/tenants/{tenant_id}/health-check") +def tenant_health_check(t: Tenant = Depends(tenant_or_404)): + """Health-Check nachholen (z. B. nach neuem Ingress, wenn das Zertifikat inzwischen da ist). Schreibt nichts außer dem Deploy-Status.""" + return deploy.recheck_health(t.id) + + @router.get("/tenants/{tenant_id}/plan") def plan(t: Tenant = Depends(tenant_or_404), db: Session = Depends(get_db)): return deploy.plan(db, t) diff --git a/app/services/deploy.py b/app/services/deploy.py index b45d1a1..f32831f 100644 --- a/app/services/deploy.py +++ b/app/services/deploy.py @@ -135,10 +135,38 @@ def _client(live: bool, log, http=None) -> Mittwald: return Mittwald(os.getenv("MITTWALD_API_TOKEN", "").strip(), os.getenv("MITTWALD_API_BASE", "https://api.mittwald.de/v2"), live, log, http) +def recheck_health(tenant_id: str, *, health=None, tries: int = 8, delay: float = 5.0) -> dict: + """Health-Check nachholen (z. B. wenn das Zertifikat inzwischen da ist). Bei Erfolg wird ein hängender Deploy-Eintrag auf „success“ gesetzt.""" + lines: list[str] = [] + with session() as db: + t = db.get(Tenant, tenant_id) + host, tag, ver, source = host_for(db, t), effective_tag(db, t), t.current_version, t.source + deployed = (t.deployed_version, t.deployed_image) + try: + args = (host, deployed[1] or tag, ver if source == "admin" else None, lines.append) + health(*args) if health else verify_health(*args, tries=tries, delay=delay) + except DeployError as exc: + return {"ok": False, "log": lines + [f"✘ {exc}"]} + with session() as db: + d = db.scalar(select(Deployment).where(Deployment.tenant_id == tenant_id, Deployment.kind == "deploy").order_by(Deployment.id.desc())) + if d is not None and d.status in ("warning", "failed") and not d.simulated: + d.status = "success" + d.log += f"{datetime.now(timezone.utc).strftime('%H:%M:%S')} Health nachträglich ✔ – Deploy als erfolgreich markiert\n" + return {"ok": True, "log": lines} + + def _hex_tag(tag: str) -> bool: return bool(tag) and all(c in "0123456789abcdef" for c in tag.lower()) and len(tag) >= 7 +class HealthTimeout(DeployError): + """Health-Check nicht grün geworden. answered=False: es kam nie eine gültige Antwort (Zertifikat/DNS/Container noch nicht bereit).""" + + def __init__(self, msg: str, answered: bool): + super().__init__(msg) + self.answered = answered + + def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, log, *, get=None, tries: int = 18, delay: float = 5.0, sleep=time.sleep) -> None: """Prüft nach dem Deploy, ob wirklich läuft, was wir ausgerollt haben (Vertrag: /health.core / .bundle). @@ -149,10 +177,11 @@ def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, def get(url): r = httpx.get(url, timeout=8, follow_redirects=True) return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {}) - last = "keine Antwort" + last, answered = "keine Antwort", False for i in range(tries): try: status, body = get(f"https://{host}/health") + answered = answered or (status == 200 and isinstance(body, dict) and "status" in body) # echte Antwort der App (nicht Proxy/Zertifikatsfehler) problems = [] if status != 200 or body.get("status") != "ok": problems.append(f"HTTP {status}") @@ -176,7 +205,7 @@ def verify_health(host: str, expect_sha: str | None, expect_bundle: int | None, last = f"{type(exc).__name__}: {exc}" if i < tries - 1: sleep(delay) - raise DeployError(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}") + raise HealthTimeout(f"Health-Check fehlgeschlagen nach {tries} Versuchen: {last}", answered) def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, core_tag: str | None = None, health=None) -> bool: @@ -230,16 +259,32 @@ def execute(dep_id: int, tenant_id: str, kind: str, http=None, oidc_client=None, else: mw.deploy_service(stack_id=sid, service=f"app-{tenant_id}", volume=f"vol-{tenant_id}", definition=definition, container_port=PORT, project_id=project, hostname=host) - if live and not kind.startswith("offboard") and host: - (health or verify_health)(host, tag, ver if source == "admin" else None, log) - if not dry: + warning = False + if not dry: # Zustand sofort festhalten: geschrieben ist geschrieben, auch wenn der Health-Check noch wartet with session() as db: t = db.get(Tenant, tenant_id) if kind == "deploy": t.deployed_version, t.deployed_image, t.deployed_at = ver, tag, datetime.now(timezone.utc) store.audit(db, "system", f"{kind}.live", tenant_id, f"v{ver} :{tag}") - log("✔ Fertig" if live else "Dry-Run beendet – es wurde nichts geschrieben.") - return True + if live and not kind.startswith("offboard") and host: + fresh = mw.ingress_created + # Neuer Ingress: Zertifikat/DNS bei Mittwald brauchen oft mehrere Minuten → geduldig warten (bis ~15 min) + kw = {"tries": 90, "delay": 10.0} if fresh else {} + if fresh: + log("Neuer Ingress angelegt – Zertifikat/DNS bei Mittwald können einige Minuten dauern, Health-Check wartet geduldig …") + try: + args = (host, tag, ver if source == "admin" else None, log) + health(*args) if health else verify_health(*args, **kw) + except HealthTimeout as exc: + if fresh and not exc.answered: # nur „noch nicht erreichbar“ bei neuem Ingress: Deploy ist durch, Prüfung später wiederholen + warning = True + log(f"⚠ {exc}") + log("⚠ Container und Ingress sind angelegt, aber die Adresse antwortet noch nicht (Zertifikat/DNS?). " + "Später „Health erneut prüfen“ (Deploy-Tab) oder „Läuft?“ (Übersicht).") + else: + raise + log("✔ Fertig" if live and not warning else ("Fertig mit Warnung" if warning else "Dry-Run beendet – es wurde nichts geschrieben.")) + return "warning" if warning else True except DeployError as exc: log(f"✘ {exc}") return False @@ -260,7 +305,7 @@ def start(tenant_id: str, kind: str, actor: str, batch_id: str = "", http=None, def work(): ok = execute(dep_id, tenant_id, kind, http, oidc_client, core_tag, health) - _finish(dep_id, "success" if ok else "failed") + _finish(dep_id, "warning" if ok == "warning" else ("success" if ok else "failed")) th = threading.Thread(target=work, daemon=True) th.start() @@ -303,8 +348,8 @@ def run_sequence(tenant_ids: list[str], kind: str, actor: str, *, batch: str, co with session() as db: db.get(Deployment, did).status = "running" ok = execute(did, tid, kind, http, oidc_client, core_tag, health) - _finish(did, "success" if ok else "failed") - results[tid] = "success" if ok else "failed" + _finish(did, "warning" if ok == "warning" else ("success" if ok else "failed")) + results[tid] = "warning" if ok == "warning" else ("success" if ok else "failed") failed = not ok if tid in canary and ok and tid == canary[-1]: _append(did, "Canary erfolgreich – restliche Tenants folgen") diff --git a/app/services/mittwald.py b/app/services/mittwald.py index ec1d73e..ab77130 100644 --- a/app/services/mittwald.py +++ b/app/services/mittwald.py @@ -30,6 +30,7 @@ class Mittwald: self.token, self.base, self.live, self.log = token, base.rstrip("/"), live, log self._http = http or self._real_http self._project_uuid: str | None = None + self.ingress_created = False # wurde in diesem Lauf ein NEUER Ingress angelegt? (dann dauert Zertifikat/DNS) @staticmethod def _real_http(method, url, headers, body): @@ -151,6 +152,8 @@ class Mittwald: body = {"hostname": hostname, "projectId": self.project_uuid(project_id), "paths": [{"path": "/", "target": {"container": {"id": container_id, "portProtocol": f"{port}/tcp"}}}]} res = self.try_api("POST", "/ingresses", body, write=True, fatal=False) + if self.live and res is not None: + self.ingress_created = True if self.live: self.log(f"Ingress {hostname} angelegt (Auto-TLS)" if res is not None else f"⚠ Ingress {hostname} NICHT angelegt – Token braucht Domain/Ingress-Recht (Service läuft trotzdem).") diff --git a/app/static/tenant.js b/app/static/tenant.js index 9362188..305f543 100644 --- a/app/static/tenant.js +++ b/app/static/tenant.js @@ -565,9 +565,9 @@ async function sDeploy(keepConsole){ OIDC: ${plan.oidc.mode}${plan.oidc.mode!=='off'&&!plan.oidc.key?' · API-Key fehlt':''} ${plan.missing_secrets.length?``:''}
Service-Definition (PATCH-Body, Secrets maskiert)
${esc(JSON.stringify(plan.patch,null,2))}
-
+
-

Verlauf

${mine.length?`${mine.map(d=>``).join('')}
ZeitArtVersionStatus
${fmt(d.created_at)}${kindLabel[d.kind]||d.kind} ${d.simulated?'Dry-Run':'live'}v${d.version}${d.status}
`:'
Noch keine Deployments für diesen Tenant.
'}
`; +

Verlauf

${mine.length?`${mine.map(d=>``).join('')}
ZeitArtVersionStatus
${fmt(d.created_at)}${kindLabel[d.kind]||d.kind} ${d.simulated?'Dry-Run':'live'}v${d.version}${d.status}
`:'
Noch keine Deployments für diesen Tenant.
'}
`; if(old){ $('#con').classList.remove('hide'); $('#con').innerHTML = old; } } async function runPreflight(){ @@ -579,6 +579,10 @@ async function runPreflight(){ r.items.map(i=>`
${ic[i.level]}${esc(i.label)} ${esc(i.detail)}${i.fix?`
→ ${esc(i.fix)}
`:''}
`).join(''); }catch(e){ $('#pf').innerHTML = ``; } } +async function recheckHealth(){ + const con = $('#con'); con.classList.remove('hide'); con.style.display='block'; con.textContent = 'Prüfe https://' + S.host + '/health … (bis ca. 40 s)'; + try{ const r = await api('POST', A('/health-check')); con.innerHTML = esc(r.log.join('\n')) + (r.ok ? '\n✔ Health ok' : ''); if(r.ok){ await load(true); sDeploy(true); } }catch(e){ con.textContent = e.message; } +} async function rollbackTenant(){ if(!confirm('Auf den vorherigen Core-Tag zurückrollen?')) return; try{ const r = await api('POST', A('/rollback')); toast('Rollback auf :'+String(r.tag).slice(0,10)+' gestartet'); followDeploy(r.id, $('#con')); }catch(e){ toast(e.message,1) } } async function startDeploy(dry){ if(!dry && !confirm('Tenant jetzt auf Mittwald deployen?')) return; diff --git a/app/templates/core.html b/app/templates/core.html index 0bc67fb..bc431e4 100644 --- a/app/templates/core.html +++ b/app/templates/core.html @@ -80,7 +80,7 @@ function followBatch(batch, title){ document.getElementById('roll-title').textContent = title; document.getElementById('roll-hint').textContent = 'Reihenfolge: Canary zuerst, dann der Rest; Abbruch beim ersten Fehler.'; modal('m-roll', true); const tick = async () => { const rows = await api('GET', '/api/deployments?batch=' + batch); - document.querySelector('#roll-table tbody').innerHTML = rows.map(d=>{ const last=(d.log.trim().split('\n').filter(l=>!l.includes('[DRY-RUN]')&&!l.includes('body:')).pop()||'').slice(10); const cls=d.status==='success'?'ok':d.status==='failed'?'err':d.status==='running'?'cyan':''; return `${esc(d.tenant)}:${esc(String(d.image).slice(0,10))}${d.status}${esc(last)}`; }).join(''); + document.querySelector('#roll-table tbody').innerHTML = rows.map(d=>{ const last=(d.log.trim().split('\n').filter(l=>!l.includes('[DRY-RUN]')&&!l.includes('body:')).pop()||'').slice(10); const cls=d.status==='success'?'ok':d.status==='failed'?'err':d.status==='warning'?'warn':d.status==='running'?'cyan':''; return `${esc(d.tenant)}:${esc(String(d.image).slice(0,10))}${d.status}${esc(last)}`; }).join(''); if(rows.some(d=>d.status==='running'||d.status==='queued')) setTimeout(tick, 800); }; tick(); } diff --git a/app/templates/dashboard.html b/app/templates/dashboard.html index 584369f..9d468f8 100644 --- a/app/templates/dashboard.html +++ b/app/templates/dashboard.html @@ -50,7 +50,7 @@

Letzte Deployments · alle ansehen

{% if last %}{% for d in last %} - {% endfor %}
{{ d.created_at|dt }}{{ d.tenant_id or 'alle' }}{{ d.kind }}v{{ d.version }}{{ d.status }}
+ {{ d.status }}{% endfor %} {% else %}
Noch keine Deployments.
{% endif %}
@@ -114,7 +114,7 @@ async function deployAll(dry){ const rows=await api('GET','/api/deployments?batch='+r.batch); document.querySelector('#all-table tbody').innerHTML=rows.map(d=>{ const last=(d.log.trim().split('\n').filter(l=>!l.includes('[DRY-RUN]')&&!l.includes('body:')).pop()||'').slice(10); - const cls=d.status==='success'?'ok':d.status==='failed'?'err':d.status==='running'?'cyan':''; + const cls=d.status==='success'?'ok':d.status==='failed'?'err':d.status==='warning'?'warn':d.status==='running'?'cyan':''; return `${esc(d.tenant)}${d.status}${esc(last)}`}).join(''); if(rows.some(d=>d.status==='running'||d.status==='queued')) setTimeout(tick,700); };tick(); diff --git a/app/templates/deployments.html b/app/templates/deployments.html index 7f104a1..9f55237 100644 --- a/app/templates/deployments.html +++ b/app/templates/deployments.html @@ -20,7 +20,7 @@ async function load(){ const rows=await api('GET','/api/deployments'); document.getElementById('list').innerHTML=rows.length?``+rows.map(d=>` - + `).join('')+'
#ZeitTenantArtVersionImageStatus
${d.id}${fmt(d.created_at)}${esc(d.tenant||'alle')}${d.kind} ${d.simulated?'Dry-Run':'live'}v${d.version??'–'}:${esc(d.image)}${d.status}
${d.status}
${esc(d.log||'(leer)')}
':'
Noch keine Deployments.
'; if(rows.some(d=>d.status==='running'||d.status==='queued')) setTimeout(load,900); } diff --git a/tests/test_core_contract.py b/tests/test_core_contract.py index 733f70d..9b8af95 100644 --- a/tests/test_core_contract.py +++ b/tests/test_core_contract.py @@ -511,3 +511,52 @@ def test_http_clients_follow_redirects_and_3xx_is_never_success(monkeypatch): # Eine nicht gefolgte 308-Weiterleitung (leerer Body) ist ein Fehler, kein „null“ with pytest.raises(DeployError, match="308"): Mittwald("t", "https://api.test/v2", False, lambda m: None, lambda *a: (308, None, "")).api("GET", "/projects/p/stacks/", write=False) + + +# ── Health-Check bei neuem Ingress (Zertifikat/DNS dauert) ────────────────── + +def test_verify_health_distinguishes_unreachable_from_wrong_version(): + from app.services.deploy import HealthTimeout + with pytest.raises(HealthTimeout) as e1: + deploy.verify_health("h", "abcdef1", None, lambda m: None, get=lambda u: (_ for _ in ()).throw(OSError("ssl handshake")), tries=2, delay=0, sleep=lambda s: None) + assert e1.value.answered is False + with pytest.raises(HealthTimeout) as e2: + deploy.verify_health("h", "abcdef1", None, lambda m: None, get=lambda u: (502, {}), tries=2, delay=0, sleep=lambda s: None) + assert e2.value.answered is False # Proxy antwortet, App nicht + with pytest.raises(HealthTimeout) as e3: + deploy.verify_health("h", "abcdef1", None, lambda m: None, get=lambda u: (200, health_body("9999999aaaa")), tries=1) + assert e3.value.answered is True + + +def test_new_ingress_with_slow_certificate_ends_with_warning_not_failure(client, live_env): + from app.services.deploy import HealthTimeout + _ready(client, "hc1") + mw = FakeMW() # GET /ingresses → [] ⇒ Ingress wird neu angelegt + never = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("keine Antwort", answered=False)) # noqa: E731 + dep = deploy.start("hc1", "deploy", "t", http=mw, health=never, wait=True) + d = client.get(f"/api/deployments/{dep}").json() + assert d["status"] == "warning" and "Zertifikat/DNS" in d["log"] and "Fertig mit Warnung" in d["log"] + with session() as db: + t = db.get(Tenant, "hc1") + assert t.deployed_version == t.current_version and t.deployed_image # Zustand wurde trotzdem festgehalten + # Zertifikat inzwischen da → nachträglicher Check macht den Eintrag grün + r = deploy.recheck_health("hc1", health=lambda *a: None) + assert r["ok"] and client.get(f"/api/deployments/{dep}").json()["status"] == "success" + + +def test_wrong_version_or_existing_ingress_still_fails(client, live_env): + from app.services.deploy import HealthTimeout + _ready(client, "hc2") + mw = FakeMW() + wrong = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("Core falsch", answered=True)) # noqa: E731 + d = client.get(f"/api/deployments/{deploy.start('hc2', 'deploy', 't', http=mw, health=wrong, wait=True)}").json() + assert d["status"] == "failed" # neuer Ingress, aber echte Antwort mit falscher Version + # bestehender Ingress + keine Antwort ⇒ echter Fehler + class WithIngress(FakeMW): + def __call__(self, method, url, headers, body): + if method == "GET" and "ingresses" in url: + return 200, [{"hostname": "hc2.onyscribe.de", "id": "i1"}], "" + return super().__call__(method, url, headers, body) + never = lambda *a, **k: (_ for _ in ()).throw(HealthTimeout("keine Antwort", answered=False)) # noqa: E731 + d2 = client.get(f"/api/deployments/{deploy.start('hc2', 'deploy', 't', http=WithIngress(), health=never, wait=True)}").json() + assert d2["status"] == "failed"