7d8c226da2
Admin-Plattform für die Tenants von Onyscribe: Editor mit Live-Vorschau (echter Core in Sandbox), Versionen, KI-Agent (Cortecs), OIDC-Abgleich über die Login-Developer-API, Deploy-Engine (Mittwald, gesperrt ohne DEPLOY_LIVE, Freigabe pro Tenant), Core-Vertrag (Schema/Manifest), Core-Releases mit Kompatibilitätsprüfung, Canary-Rollout und Rollback, Zustandsabgleich. Runbook: docs/RUNBOOK_PARALLELTEST.md Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
101 lines
4.1 KiB
Python
101 lines
4.1 KiB
Python
"""Lightweight retry helpers for transient provider failures.
|
|
|
|
The transcription (Mistral) and e-mail (SMTP) call sites had no retry at all, so
|
|
a brief provider outage aborted the whole job. These helpers add exponential
|
|
backoff with jitter for *transient* failures only — network errors, timeouts and
|
|
5xx/429 — while failing fast on permanent errors (bad request, auth, payload),
|
|
where retrying cannot help.
|
|
|
|
The OpenAI SDK used for Cortecs already retries internally (configured via
|
|
LLM_MAX_RETRIES at the client); this module covers the two raw call sites.
|
|
|
|
All knobs are env-tunable so they can be adjusted without a redeploy:
|
|
RETRY_MAX_ATTEMPTS total attempts incl. the first (default 4)
|
|
RETRY_BASE_DELAY base backoff in seconds (default 1.0)
|
|
RETRY_MAX_DELAY cap per wait in seconds (default 30.0)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import random
|
|
import time
|
|
|
|
import requests
|
|
|
|
RETRY_MAX_ATTEMPTS = int(os.getenv("RETRY_MAX_ATTEMPTS", "4"))
|
|
RETRY_BASE_DELAY = float(os.getenv("RETRY_BASE_DELAY", "1.0"))
|
|
RETRY_MAX_DELAY = float(os.getenv("RETRY_MAX_DELAY", "30.0"))
|
|
|
|
# Transient HTTP statuses — worth retrying.
|
|
RETRYABLE_STATUS = {408, 425, 429, 500, 502, 503, 504}
|
|
# Network-level failures before any HTTP response arrives.
|
|
RETRYABLE_HTTP_EXC = (requests.exceptions.ConnectionError, requests.exceptions.Timeout)
|
|
|
|
|
|
def _backoff_delay(attempt: int, retry_after: float | None) -> float:
|
|
"""Seconds to wait before the next attempt (0-indexed)."""
|
|
if retry_after is not None:
|
|
return min(retry_after, RETRY_MAX_DELAY)
|
|
# exponential backoff with full jitter
|
|
ceiling = min(RETRY_BASE_DELAY * (2 ** attempt), RETRY_MAX_DELAY)
|
|
return random.uniform(0, ceiling)
|
|
|
|
|
|
def _retry_after_seconds(resp: requests.Response) -> float | None:
|
|
val = resp.headers.get("Retry-After")
|
|
if not val:
|
|
return None
|
|
try:
|
|
return float(val) # delta-seconds form
|
|
except ValueError:
|
|
return None # HTTP-date form — fall back to backoff
|
|
|
|
|
|
def request_with_retry(do_request, *, label: str = "request") -> requests.Response:
|
|
"""Call ``do_request()`` (a zero-arg callable returning a ``requests.Response``)
|
|
with retries on transient network errors and 5xx/429.
|
|
|
|
The caller still inspects the returned response for non-retryable ``>= 400``
|
|
statuses. ``do_request`` must rebuild any per-attempt state (e.g. re-open the
|
|
upload file) so each retry sends a complete request.
|
|
"""
|
|
for attempt in range(RETRY_MAX_ATTEMPTS):
|
|
last = attempt == RETRY_MAX_ATTEMPTS - 1
|
|
try:
|
|
resp = do_request()
|
|
except RETRYABLE_HTTP_EXC as exc:
|
|
if last:
|
|
raise
|
|
delay = _backoff_delay(attempt, None)
|
|
print(f"[retry] {label}: {type(exc).__name__} "
|
|
f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s")
|
|
time.sleep(delay)
|
|
continue
|
|
if resp.status_code in RETRYABLE_STATUS and not last:
|
|
delay = _backoff_delay(attempt, _retry_after_seconds(resp))
|
|
print(f"[retry] {label}: HTTP {resp.status_code} "
|
|
f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s")
|
|
try:
|
|
resp.close()
|
|
except Exception:
|
|
pass
|
|
time.sleep(delay)
|
|
continue
|
|
return resp
|
|
# Unreachable: the loop either returns a response or raises on the last attempt.
|
|
raise RuntimeError(f"{label}: retry loop ended without a result")
|
|
|
|
|
|
def call_with_retry(fn, *, retry_on: tuple[type[BaseException], ...], label: str = "call"):
|
|
"""Retry an arbitrary callable on the given (transient) exception types."""
|
|
for attempt in range(RETRY_MAX_ATTEMPTS):
|
|
try:
|
|
return fn()
|
|
except retry_on as exc:
|
|
if attempt == RETRY_MAX_ATTEMPTS - 1:
|
|
raise
|
|
delay = _backoff_delay(attempt, None)
|
|
print(f"[retry] {label}: {type(exc).__name__} "
|
|
f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s")
|
|
time.sleep(delay)
|