Onyscribe Admin: Initialer Stand
Admin-Plattform für die Tenants von Onyscribe: Editor mit Live-Vorschau (echter Core in Sandbox), Versionen, KI-Agent (Cortecs), OIDC-Abgleich über die Login-Developer-API, Deploy-Engine (Mittwald, gesperrt ohne DEPLOY_LIVE, Freigabe pro Tenant), Core-Vertrag (Schema/Manifest), Core-Releases mit Kompatibilitätsprüfung, Canary-Rollout und Rollback, Zustandsabgleich. Runbook: docs/RUNBOOK_PARALLELTEST.md Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
100
vendor/platform/core/retry.py
vendored
Normal file
100
vendor/platform/core/retry.py
vendored
Normal file
@@ -0,0 +1,100 @@
|
||||
"""Lightweight retry helpers for transient provider failures.
|
||||
|
||||
The transcription (Mistral) and e-mail (SMTP) call sites had no retry at all, so
|
||||
a brief provider outage aborted the whole job. These helpers add exponential
|
||||
backoff with jitter for *transient* failures only — network errors, timeouts and
|
||||
5xx/429 — while failing fast on permanent errors (bad request, auth, payload),
|
||||
where retrying cannot help.
|
||||
|
||||
The OpenAI SDK used for Cortecs already retries internally (configured via
|
||||
LLM_MAX_RETRIES at the client); this module covers the two raw call sites.
|
||||
|
||||
All knobs are env-tunable so they can be adjusted without a redeploy:
|
||||
RETRY_MAX_ATTEMPTS total attempts incl. the first (default 4)
|
||||
RETRY_BASE_DELAY base backoff in seconds (default 1.0)
|
||||
RETRY_MAX_DELAY cap per wait in seconds (default 30.0)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
RETRY_MAX_ATTEMPTS = int(os.getenv("RETRY_MAX_ATTEMPTS", "4"))
|
||||
RETRY_BASE_DELAY = float(os.getenv("RETRY_BASE_DELAY", "1.0"))
|
||||
RETRY_MAX_DELAY = float(os.getenv("RETRY_MAX_DELAY", "30.0"))
|
||||
|
||||
# Transient HTTP statuses — worth retrying.
|
||||
RETRYABLE_STATUS = {408, 425, 429, 500, 502, 503, 504}
|
||||
# Network-level failures before any HTTP response arrives.
|
||||
RETRYABLE_HTTP_EXC = (requests.exceptions.ConnectionError, requests.exceptions.Timeout)
|
||||
|
||||
|
||||
def _backoff_delay(attempt: int, retry_after: float | None) -> float:
|
||||
"""Seconds to wait before the next attempt (0-indexed)."""
|
||||
if retry_after is not None:
|
||||
return min(retry_after, RETRY_MAX_DELAY)
|
||||
# exponential backoff with full jitter
|
||||
ceiling = min(RETRY_BASE_DELAY * (2 ** attempt), RETRY_MAX_DELAY)
|
||||
return random.uniform(0, ceiling)
|
||||
|
||||
|
||||
def _retry_after_seconds(resp: requests.Response) -> float | None:
|
||||
val = resp.headers.get("Retry-After")
|
||||
if not val:
|
||||
return None
|
||||
try:
|
||||
return float(val) # delta-seconds form
|
||||
except ValueError:
|
||||
return None # HTTP-date form — fall back to backoff
|
||||
|
||||
|
||||
def request_with_retry(do_request, *, label: str = "request") -> requests.Response:
|
||||
"""Call ``do_request()`` (a zero-arg callable returning a ``requests.Response``)
|
||||
with retries on transient network errors and 5xx/429.
|
||||
|
||||
The caller still inspects the returned response for non-retryable ``>= 400``
|
||||
statuses. ``do_request`` must rebuild any per-attempt state (e.g. re-open the
|
||||
upload file) so each retry sends a complete request.
|
||||
"""
|
||||
for attempt in range(RETRY_MAX_ATTEMPTS):
|
||||
last = attempt == RETRY_MAX_ATTEMPTS - 1
|
||||
try:
|
||||
resp = do_request()
|
||||
except RETRYABLE_HTTP_EXC as exc:
|
||||
if last:
|
||||
raise
|
||||
delay = _backoff_delay(attempt, None)
|
||||
print(f"[retry] {label}: {type(exc).__name__} "
|
||||
f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if resp.status_code in RETRYABLE_STATUS and not last:
|
||||
delay = _backoff_delay(attempt, _retry_after_seconds(resp))
|
||||
print(f"[retry] {label}: HTTP {resp.status_code} "
|
||||
f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s")
|
||||
try:
|
||||
resp.close()
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(delay)
|
||||
continue
|
||||
return resp
|
||||
# Unreachable: the loop either returns a response or raises on the last attempt.
|
||||
raise RuntimeError(f"{label}: retry loop ended without a result")
|
||||
|
||||
|
||||
def call_with_retry(fn, *, retry_on: tuple[type[BaseException], ...], label: str = "call"):
|
||||
"""Retry an arbitrary callable on the given (transient) exception types."""
|
||||
for attempt in range(RETRY_MAX_ATTEMPTS):
|
||||
try:
|
||||
return fn()
|
||||
except retry_on as exc:
|
||||
if attempt == RETRY_MAX_ATTEMPTS - 1:
|
||||
raise
|
||||
delay = _backoff_delay(attempt, None)
|
||||
print(f"[retry] {label}: {type(exc).__name__} "
|
||||
f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
Reference in New Issue
Block a user