"""Lightweight retry helpers for transient provider failures. The transcription (Mistral) and e-mail (SMTP) call sites had no retry at all, so a brief provider outage aborted the whole job. These helpers add exponential backoff with jitter for *transient* failures only — network errors, timeouts and 5xx/429 — while failing fast on permanent errors (bad request, auth, payload), where retrying cannot help. The OpenAI SDK used for Cortecs already retries internally (configured via LLM_MAX_RETRIES at the client); this module covers the two raw call sites. All knobs are env-tunable so they can be adjusted without a redeploy: RETRY_MAX_ATTEMPTS total attempts incl. the first (default 4) RETRY_BASE_DELAY base backoff in seconds (default 1.0) RETRY_MAX_DELAY cap per wait in seconds (default 30.0) """ from __future__ import annotations import os import random import time import requests RETRY_MAX_ATTEMPTS = int(os.getenv("RETRY_MAX_ATTEMPTS", "4")) RETRY_BASE_DELAY = float(os.getenv("RETRY_BASE_DELAY", "1.0")) RETRY_MAX_DELAY = float(os.getenv("RETRY_MAX_DELAY", "30.0")) # Transient HTTP statuses — worth retrying. RETRYABLE_STATUS = {408, 425, 429, 500, 502, 503, 504} # Network-level failures before any HTTP response arrives. RETRYABLE_HTTP_EXC = (requests.exceptions.ConnectionError, requests.exceptions.Timeout) def _backoff_delay(attempt: int, retry_after: float | None) -> float: """Seconds to wait before the next attempt (0-indexed).""" if retry_after is not None: return min(retry_after, RETRY_MAX_DELAY) # exponential backoff with full jitter ceiling = min(RETRY_BASE_DELAY * (2 ** attempt), RETRY_MAX_DELAY) return random.uniform(0, ceiling) def _retry_after_seconds(resp: requests.Response) -> float | None: val = resp.headers.get("Retry-After") if not val: return None try: return float(val) # delta-seconds form except ValueError: return None # HTTP-date form — fall back to backoff def request_with_retry(do_request, *, label: str = "request") -> requests.Response: """Call ``do_request()`` (a zero-arg callable returning a ``requests.Response``) with retries on transient network errors and 5xx/429. The caller still inspects the returned response for non-retryable ``>= 400`` statuses. ``do_request`` must rebuild any per-attempt state (e.g. re-open the upload file) so each retry sends a complete request. """ for attempt in range(RETRY_MAX_ATTEMPTS): last = attempt == RETRY_MAX_ATTEMPTS - 1 try: resp = do_request() except RETRYABLE_HTTP_EXC as exc: if last: raise delay = _backoff_delay(attempt, None) print(f"[retry] {label}: {type(exc).__name__} " f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s") time.sleep(delay) continue if resp.status_code in RETRYABLE_STATUS and not last: delay = _backoff_delay(attempt, _retry_after_seconds(resp)) print(f"[retry] {label}: HTTP {resp.status_code} " f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s") try: resp.close() except Exception: pass time.sleep(delay) continue return resp # Unreachable: the loop either returns a response or raises on the last attempt. raise RuntimeError(f"{label}: retry loop ended without a result") def call_with_retry(fn, *, retry_on: tuple[type[BaseException], ...], label: str = "call"): """Retry an arbitrary callable on the given (transient) exception types.""" for attempt in range(RETRY_MAX_ATTEMPTS): try: return fn() except retry_on as exc: if attempt == RETRY_MAX_ATTEMPTS - 1: raise delay = _backoff_delay(attempt, None) print(f"[retry] {label}: {type(exc).__name__} " f"(Versuch {attempt + 1}/{RETRY_MAX_ATTEMPTS}), warte {delay:.1f}s") time.sleep(delay)