lead-scraper/lib/services/anymailfinder.ts

// Anymailfinder API v5.1
// Docs: https://anymailfinder.com/api
// Auth: Authorization: YOUR_API_KEY (header)
// No rate limits on individual searches.
// Bulk API processes ~1,000 rows per 5 minutes asynchronously.

import axios from "axios";
import { extractDomainFromUrl } from "@/lib/utils/domains";

const BASE_URL = "https://api.anymailfinder.com/v5.1";

export type DecisionMakerCategory =
  | "ceo" | "engineering" | "finance" | "hr" | "it"
  | "logistics" | "marketing" | "operations" | "buyer" | "sales";

export interface DecisionMakerResult {
  decision_maker_category: string | null;
  email: string | null;
  email_status: "valid" | "risky" | "not_found" | "blacklisted";
  person_full_name: string | null;
  person_job_title: string | null;
  person_linkedin_url: string | null;
  valid_email: string | null;
  domain?: string;
}

export interface BulkSearchResult {
  domain: string;
  email: string | null;
  email_status: string;
  person_full_name: string | null;
  person_job_title: string | null;
  valid_email: string | null;
}

// ─── Individual search (used for small batches / LinkedIn enrichment) ─────────

export async function searchDecisionMakerByDomain(
  domain: string,
  categories: DecisionMakerCategory[],
  apiKey: string
): Promise<DecisionMakerResult> {
  const response = await axios.post(
    `${BASE_URL}/find-email/decision-maker`,
    { domain, decision_maker_category: categories },
    {
      headers: { Authorization: apiKey, "Content-Type": "application/json" },
      timeout: 180000,
    }
  );
  return { ...response.data, domain };
}

// ─── Bulk JSON search (preferred for large domain lists) ────────────────────

export interface BulkJobStatus {
  id: string;
  status: "queued" | "running" | "completed" | "failed" | "paused" | "on_deck";
  counts: {
    total: number;
    found_valid: number;
    found_unknown: number;
    not_found: number;
    failed: number;
  };
}

/**
 * Submit a bulk decision-maker search via the JSON API.
 * Returns a searchId to poll for completion.
 */
export async function submitBulkDecisionMakerSearch(
  domains: string[],
  category: DecisionMakerCategory,
  apiKey: string,
  fileName?: string
): Promise<string> {
  // Build data array: header row + data rows
  const data: string[][] = [
    ["domain"],
    ...domains.map(d => [d]),
  ];

  const response = await axios.post(
    `${BASE_URL}/bulk/json`,
    {
      data,
      domain_field_index: 0,
      decision_maker_category: category,
      file_name: fileName || `onyva-leads-${Date.now()}`,
    },
    {
      headers: { Authorization: apiKey, "Content-Type": "application/json" },
      timeout: 30000,
    }
  );
  return response.data.id as string;
}

/**
 * Submit a bulk person name search via the JSON API.
 * Used for LinkedIn enrichment where we have names + domains.
 */
export async function submitBulkPersonSearch(
  leads: Array<{ domain: string; firstName: string; lastName: string }>,
  apiKey: string,
  fileName?: string
): Promise<string> {
  const data: string[][] = [
    ["domain", "first_name", "last_name"],
    ...leads.map(l => [l.domain, l.firstName, l.lastName]),
  ];

  const response = await axios.post(
    `${BASE_URL}/bulk/json`,
    {
      data,
      domain_field_index: 0,
      first_name_field_index: 1,
      last_name_field_index: 2,
      file_name: fileName || `onyva-leads-${Date.now()}`,
    },
    {
      headers: { Authorization: apiKey, "Content-Type": "application/json" },
      timeout: 30000,
    }
  );
  return response.data.id as string;
}

export async function getBulkSearchStatus(
  searchId: string,
  apiKey: string
): Promise<BulkJobStatus> {
  const response = await axios.get(`${BASE_URL}/bulk/${searchId}`, {
    headers: { Authorization: apiKey },
    timeout: 15000,
  });
  return response.data;
}

/**
 * Download bulk search results as JSON array.
 * IMPORTANT: Credits are charged on first download.
 * Handles both array-of-objects and array-of-arrays (with header row) formats.
 */
export async function downloadBulkResults(
  searchId: string,
  apiKey: string
): Promise<Array<Record<string, string>>> {
  const response = await axios.get(`${BASE_URL}/bulk/${searchId}/download`, {
    params: { download_as: "json_arr" },
    headers: { Authorization: apiKey },
    timeout: 60000,
  });

  const raw = response.data;

  // Handle array-of-arrays format (first row = headers)
  if (Array.isArray(raw) && Array.isArray(raw[0])) {
    const headers = (raw[0] as string[]).map(h => h.toLowerCase().trim());
    return (raw as string[][]).slice(1).map(row => {
      const obj: Record<string, string> = {};
      headers.forEach((h, i) => { obj[h] = row[i] ?? ""; });
      return obj;
    });
  }

  // Standard array-of-objects format
  return raw as Array<Record<string, string>>;
}

/**
 * High-level bulk enrichment: submit → poll → download → return results.
 * Uses the Bulk JSON API for efficiency (1,000 rows/5 min).
 * Calls onProgress with status updates while waiting.
 */
export async function bulkSearchDomains(
  domains: string[],
  categories: DecisionMakerCategory[],
  apiKey: string,
  onProgress?: (completed: number, total: number, result?: DecisionMakerResult) => Promise<void> | void
): Promise<DecisionMakerResult[]> {
  if (domains.length === 0) return [];

  // Use the primary category (first in list) for bulk search.
  // Anymailfinder bulk API takes one category at a time.
  const primaryCategory = categories[0] || "ceo";

  // 1. Submit bulk job
  const searchId = await submitBulkDecisionMakerSearch(
    domains,
    primaryCategory,
    apiKey,
    `onyva-leads-bulk-${Date.now()}`
  );

  // 2. Poll until complete (~1,000 rows per 5 min)
  let status: BulkJobStatus;
  do {
    await sleep(5000);
    status = await getBulkSearchStatus(searchId, apiKey);
    const processed = (status.counts?.found_valid || 0) + (status.counts?.not_found || 0) + (status.counts?.found_unknown || 0);
    onProgress?.(processed, status.counts?.total || domains.length);
  } while (status.status !== "completed" && status.status !== "failed");

  if (status.status === "failed") {
    throw new Error(`Anymailfinder bulk search failed (id: ${searchId})`);
  }

  // 3. Download results
  const rows = await downloadBulkResults(searchId, apiKey);

  // 4. Normalize to DecisionMakerResult[]
  // Log first row to help debug field names
  if (rows.length > 0) {
    console.log("[Anymailfinder] Raw response sample row keys:", Object.keys(rows[0]));
    console.log("[Anymailfinder] Sample row:", JSON.stringify(rows[0]));
  }

  return rows.map(row => {
    const rawDomain = row["domain"] || row["Domain"] || "";
    const domain = rawDomain ? extractDomainFromUrl(rawDomain.includes(".") ? rawDomain : `https://${rawDomain}`) : "";

    // Anymailfinder bulk JSON uses different field names than the single-search API:
    // amf_status (not email_status), valid_email_only (not valid_email),
    // person_name (not person_full_name), result_title (not person_job_title),
    // linkedin_url (not person_linkedin_url)
    const email = row["email"] || row["Email"] || null;
    const rawStatus = row["amf_status"] || row["email_status"] || row["Email Status"] || "not_found";
    // amf_status uses "not found" (space) — normalize to underscore variant
    const emailStatus = rawStatus.toLowerCase().replace(/\s+/g, "_") as DecisionMakerResult["email_status"];
    const validEmail = row["valid_email_only"] || row["valid_email"] || (emailStatus === "valid" ? email : null) || null;

    return {
      domain,
      decision_maker_category: primaryCategory,
      email: email || validEmail,
      email_status: emailStatus,
      valid_email: validEmail,
      person_full_name: row["person_name"] || row["person_full_name"] || row["Full Name"] || null,
      person_job_title: row["result_title"] || row["person_job_title"] || row["Job Title"] || null,
      person_linkedin_url: row["linkedin_url"] || row["person_linkedin_url"] || row["LinkedIn URL"] || null,
    };
  });
}

export async function getRemainingCredits(apiKey: string): Promise<number | null> {
  try {
    // Try account endpoint (may not be documented publicly, returns null if unavailable)
    const response = await axios.get(`${BASE_URL}/account`, {
      headers: { Authorization: apiKey },
      timeout: 10000,
    });
    return response.data?.credits_remaining ?? null;
  } catch {
    return null;
  }
}

function sleep(ms: number) {
  return new Promise(resolve => setTimeout(resolve, ms));
}