facf8c9f69
Full-stack Next.js 16 app with three scraping pipelines: - AirScale CSV → Anymailfinder Bulk Decision Maker search - LinkedIn Sales Navigator → Vayne → Anymailfinder email enrichment - Apify Google SERP → domain extraction → Anymailfinder bulk enrichment Includes Docker multi-stage build + docker-compose for Coolify deployment. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
420 lines
17 KiB
TypeScript
420 lines
17 KiB
TypeScript
"use client";
|
||
|
||
import { useState, useEffect } from "react";
|
||
import { Card } from "@/components/ui/card";
|
||
import { Button } from "@/components/ui/button";
|
||
import { Input } from "@/components/ui/input";
|
||
import { Label } from "@/components/ui/label";
|
||
import { Textarea } from "@/components/ui/textarea";
|
||
import { ProgressCard } from "@/components/shared/ProgressCard";
|
||
import { ResultsTable, type ResultRow } from "@/components/shared/ResultsTable";
|
||
import { ExportButtons } from "@/components/shared/ExportButtons";
|
||
import { EmptyState } from "@/components/shared/EmptyState";
|
||
import { toast } from "sonner";
|
||
import {
|
||
Linkedin, ChevronRight, AlertTriangle, CheckCircle2, XCircle,
|
||
ChevronDown, ChevronUp, Info
|
||
} from "lucide-react";
|
||
import { useAppStore } from "@/lib/store";
|
||
import type { DecisionMakerCategory } from "@/lib/services/anymailfinder";
|
||
import type { ExportRow } from "@/lib/utils/csv";
|
||
|
||
const CATEGORY_OPTIONS: { value: DecisionMakerCategory; label: string }[] = [
|
||
{ value: "ceo", label: "CEO / Owner / Founder" },
|
||
{ value: "engineering", label: "Engineering" },
|
||
{ value: "marketing", label: "Marketing" },
|
||
{ value: "sales", label: "Sales" },
|
||
{ value: "operations", label: "Operations" },
|
||
{ value: "finance", label: "Finance" },
|
||
{ value: "hr", label: "HR" },
|
||
{ value: "it", label: "IT" },
|
||
{ value: "buyer", label: "Procurement" },
|
||
{ value: "logistics", label: "Logistics" },
|
||
];
|
||
|
||
type Stage = "idle" | "scraping" | "scraped" | "enriching" | "done" | "failed";
|
||
|
||
export default function LinkedInPage() {
|
||
const [salesNavUrl, setSalesNavUrl] = useState("");
|
||
const [maxResults, setMaxResults] = useState(100);
|
||
const [vayneConfigured, setVayneConfigured] = useState<boolean | null>(null);
|
||
const [guideOpen, setGuideOpen] = useState(false);
|
||
const [stage, setStage] = useState<Stage>("idle");
|
||
const [scrapeJobId, setScrapeJobId] = useState<string | null>(null);
|
||
const [enrichJobId, setEnrichJobId] = useState<string | null>(null);
|
||
const [scrapeProgress, setScrapeProgress] = useState({ current: 0, total: 0 });
|
||
const [enrichProgress, setEnrichProgress] = useState({ current: 0, total: 0 });
|
||
const [results, setResults] = useState<ResultRow[]>([]);
|
||
const [selectedIds, setSelectedIds] = useState<string[]>([]);
|
||
const [categories, setCategories] = useState<DecisionMakerCategory[]>(["ceo"]);
|
||
const { addJob, updateJob, removeJob } = useAppStore();
|
||
|
||
useEffect(() => {
|
||
fetch("/api/credentials")
|
||
.then(r => r.json())
|
||
.then((d: Record<string, boolean>) => setVayneConfigured(d.vayne))
|
||
.catch(() => setVayneConfigured(false));
|
||
}, []);
|
||
|
||
const urlValid = salesNavUrl.includes("linkedin.com/sales");
|
||
|
||
const startScrape = async () => {
|
||
if (!urlValid) return toast.error("Please paste a valid Sales Navigator URL");
|
||
if (!vayneConfigured) return toast.error("Configure your Vayne API token in Settings first");
|
||
|
||
setStage("scraping");
|
||
setScrapeProgress({ current: 0, total: maxResults });
|
||
setResults([]);
|
||
|
||
try {
|
||
const res = await fetch("/api/jobs/vayne-scrape", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ salesNavUrl, maxResults }),
|
||
});
|
||
const data = await res.json() as { jobId?: string; error?: string };
|
||
if (!res.ok || !data.jobId) throw new Error(data.error || "Failed");
|
||
|
||
setScrapeJobId(data.jobId);
|
||
addJob({ id: data.jobId, type: "linkedin-scrape", status: "running", progress: 0, total: maxResults });
|
||
pollScrape(data.jobId);
|
||
} catch (err) {
|
||
toast.error(err instanceof Error ? err.message : "Failed to start scrape");
|
||
setStage("failed");
|
||
}
|
||
};
|
||
|
||
const pollScrape = (id: string) => {
|
||
const interval = setInterval(async () => {
|
||
try {
|
||
const res = await fetch(`/api/jobs/${id}/status`);
|
||
const data = await res.json() as {
|
||
status: string; totalLeads: number; results: ResultRow[];
|
||
};
|
||
|
||
setScrapeProgress({ current: data.totalLeads, total: maxResults });
|
||
if (data.results?.length) setResults(data.results);
|
||
|
||
updateJob(id, { status: data.status, progress: data.totalLeads });
|
||
|
||
if (data.status === "complete" || data.status === "failed") {
|
||
clearInterval(interval);
|
||
removeJob(id);
|
||
if (data.status === "complete") {
|
||
setStage("scraped");
|
||
setResults(data.results || []);
|
||
setSelectedIds(data.results?.map(r => r.id) || []);
|
||
toast.success(`Scraped ${data.totalLeads} profiles from LinkedIn`);
|
||
} else {
|
||
setStage("failed");
|
||
toast.error("Scrape failed. Check Vayne token in Settings.");
|
||
}
|
||
}
|
||
} catch {
|
||
clearInterval(interval);
|
||
setStage("failed");
|
||
}
|
||
}, 2000);
|
||
};
|
||
|
||
const startEnrich = async () => {
|
||
if (!selectedIds.length) return toast.error("Select at least one profile to enrich");
|
||
if (!categories.length) return toast.error("Select at least one category");
|
||
|
||
setStage("enriching");
|
||
setEnrichProgress({ current: 0, total: selectedIds.length });
|
||
|
||
try {
|
||
const res = await fetch("/api/jobs/linkedin-enrich", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ jobId: scrapeJobId, resultIds: selectedIds, categories }),
|
||
});
|
||
const data = await res.json() as { jobId?: string; error?: string };
|
||
if (!res.ok || !data.jobId) throw new Error(data.error || "Failed");
|
||
|
||
setEnrichJobId(data.jobId);
|
||
addJob({ id: data.jobId, type: "linkedin-enrich", status: "running", progress: 0, total: selectedIds.length });
|
||
pollEnrich(data.jobId);
|
||
} catch (err) {
|
||
toast.error(err instanceof Error ? err.message : "Failed to start enrichment");
|
||
setStage("scraped");
|
||
}
|
||
};
|
||
|
||
const pollEnrich = (id: string) => {
|
||
const interval = setInterval(async () => {
|
||
try {
|
||
const res = await fetch(`/api/jobs/${id}/status`);
|
||
const data = await res.json() as {
|
||
status: string; totalLeads: number; emailsFound: number;
|
||
};
|
||
|
||
setEnrichProgress({ current: data.emailsFound, total: data.totalLeads });
|
||
updateJob(id, { status: data.status, progress: data.emailsFound });
|
||
|
||
if (data.status === "complete" || data.status === "failed") {
|
||
clearInterval(interval);
|
||
removeJob(id);
|
||
// Refresh results from scrape job
|
||
const jobRes = await fetch(`/api/jobs/${scrapeJobId}/status`);
|
||
const jobData = await jobRes.json() as { results: ResultRow[] };
|
||
setResults(jobData.results || []);
|
||
|
||
if (data.status === "complete") {
|
||
setStage("done");
|
||
toast.success(`Found ${data.emailsFound} emails`);
|
||
} else {
|
||
setStage("scraped");
|
||
toast.error("Enrichment failed");
|
||
}
|
||
}
|
||
} catch {
|
||
clearInterval(interval);
|
||
}
|
||
}, 2000);
|
||
};
|
||
|
||
const exportRows: ExportRow[] = results.map(r => ({
|
||
company_name: r.companyName,
|
||
domain: r.domain,
|
||
contact_name: r.contactName,
|
||
contact_title: r.contactTitle,
|
||
email: r.email,
|
||
confidence_score: r.confidence !== undefined ? Math.round(r.confidence * 100) : undefined,
|
||
source_tab: "linkedin",
|
||
job_id: scrapeJobId || "",
|
||
found_at: new Date().toISOString(),
|
||
}));
|
||
|
||
const emailsFound = results.filter(r => r.email).length;
|
||
|
||
return (
|
||
<div className="space-y-6 max-w-5xl">
|
||
{/* Header */}
|
||
<div className="relative rounded-2xl bg-gradient-to-r from-blue-700/10 to-blue-500/10 border border-[#1e1e2e] p-6 overflow-hidden">
|
||
<div className="absolute inset-0 bg-gradient-to-br from-blue-600/5 to-transparent" />
|
||
<div className="relative">
|
||
<div className="flex items-center gap-2 text-sm text-blue-400 mb-2">
|
||
<Linkedin className="w-4 h-4" />
|
||
<span>Tab 2</span>
|
||
<ChevronRight className="w-3 h-3" />
|
||
<span>LinkedIn Sales Navigator</span>
|
||
</div>
|
||
<h1 className="text-2xl font-bold text-white">LinkedIn → Email Pipeline</h1>
|
||
<p className="text-gray-400 mt-1 text-sm">
|
||
Scrape Sales Navigator profiles via Vayne, then enrich with Anymailfinder.
|
||
</p>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Filter Guide */}
|
||
<Card className="bg-[#111118] border-purple-500/20 p-0 overflow-hidden">
|
||
<button
|
||
onClick={() => setGuideOpen(g => !g)}
|
||
className="w-full flex items-center justify-between px-6 py-4 hover:bg-purple-500/5 transition-colors"
|
||
>
|
||
<div className="flex items-center gap-2 text-purple-300 font-medium">
|
||
<Info className="w-4 h-4" />
|
||
Recommended Sales Navigator Filter Settings
|
||
</div>
|
||
{guideOpen ? <ChevronUp className="w-4 h-4 text-gray-500" /> : <ChevronDown className="w-4 h-4 text-gray-500" />}
|
||
</button>
|
||
{guideOpen && (
|
||
<div className="px-6 pb-5 border-t border-purple-500/10 space-y-4 text-sm">
|
||
<div className="grid grid-cols-2 gap-4 pt-4">
|
||
<div>
|
||
<p className="text-gray-400 font-medium mb-2">Keywords</p>
|
||
<div className="flex flex-wrap gap-1.5">
|
||
{["Solarlösungen", "Founder", "Co-Founder", "CEO", "Geschäftsführer"].map(k => (
|
||
<span key={k} className="bg-purple-500/10 text-purple-300 px-2 py-0.5 rounded text-xs">{k}</span>
|
||
))}
|
||
</div>
|
||
</div>
|
||
<div>
|
||
<p className="text-gray-400 font-medium mb-2">Headcount</p>
|
||
<div className="flex flex-wrap gap-1.5">
|
||
{["1–10", "11–50", "51–200"].map(k => (
|
||
<span key={k} className="bg-blue-500/10 text-blue-300 px-2 py-0.5 rounded text-xs">{k}</span>
|
||
))}
|
||
</div>
|
||
</div>
|
||
<div>
|
||
<p className="text-gray-400 font-medium mb-2">Country</p>
|
||
<span className="bg-green-500/10 text-green-300 px-2 py-0.5 rounded text-xs">Germany (Deutschland)</span>
|
||
</div>
|
||
<div>
|
||
<p className="text-gray-400 font-medium mb-2">Target Titles</p>
|
||
<div className="flex flex-wrap gap-1.5">
|
||
{["Founder", "Co-Founder", "CEO", "CTO", "COO", "Owner", "President", "Principal", "Partner"].map(k => (
|
||
<span key={k} className="bg-purple-500/10 text-purple-300 px-2 py-0.5 rounded text-xs">{k}</span>
|
||
))}
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
)}
|
||
</Card>
|
||
|
||
{/* Step 1: Input */}
|
||
<Card className="bg-[#111118] border-[#1e1e2e] p-6 space-y-5">
|
||
<h2 className="text-base font-semibold text-white flex items-center gap-2">
|
||
<span className="w-6 h-6 rounded-full bg-blue-500/20 text-blue-400 text-xs flex items-center justify-center font-bold">1</span>
|
||
Sales Navigator URL
|
||
</h2>
|
||
|
||
{/* Vayne status */}
|
||
<div className={`flex items-center gap-2 px-3 py-2.5 rounded-lg border text-sm ${
|
||
vayneConfigured === null ? "bg-[#0d0d18] border-[#1e1e2e] text-gray-400"
|
||
: vayneConfigured ? "bg-green-500/5 border-green-500/20 text-green-400"
|
||
: "bg-red-500/5 border-red-500/20 text-red-400"
|
||
}`}>
|
||
{vayneConfigured === null ? "Checking Vayne configuration..."
|
||
: vayneConfigured ? (
|
||
<><CheckCircle2 className="w-4 h-4" /> Vayne API token configured</>
|
||
) : (
|
||
<><XCircle className="w-4 h-4" /> Vayne token not configured — <a href="/settings" className="underline">go to Settings</a></>
|
||
)}
|
||
</div>
|
||
|
||
<div>
|
||
<Label className="text-gray-300 text-sm mb-1.5 block">Sales Navigator Search URL</Label>
|
||
<Textarea
|
||
placeholder="https://www.linkedin.com/sales/search/people?query=..."
|
||
value={salesNavUrl}
|
||
onChange={e => setSalesNavUrl(e.target.value)}
|
||
className="bg-[#0d0d18] border-[#2e2e3e] text-white placeholder:text-gray-600 focus:border-blue-500 resize-none h-20 font-mono text-xs"
|
||
/>
|
||
{salesNavUrl && !urlValid && (
|
||
<p className="text-xs text-red-400 mt-1">Must be a linkedin.com/sales/search URL</p>
|
||
)}
|
||
</div>
|
||
|
||
<div>
|
||
<Label className="text-gray-300 text-sm mb-1.5 block">Max results to scrape</Label>
|
||
<Input
|
||
type="number"
|
||
min={1}
|
||
max={2500}
|
||
value={maxResults}
|
||
onChange={e => setMaxResults(Number(e.target.value))}
|
||
className="bg-[#0d0d18] border-[#2e2e3e] text-white w-32"
|
||
/>
|
||
</div>
|
||
|
||
{/* Warning */}
|
||
<div className="flex items-start gap-2 bg-yellow-500/5 border border-yellow-500/20 rounded-lg px-4 py-3">
|
||
<AlertTriangle className="w-4 h-4 text-yellow-400 flex-shrink-0 mt-0.5" />
|
||
<p className="text-xs text-yellow-300">
|
||
Do not close this tab while scraping is in progress. The job runs in the background
|
||
but progress tracking requires this tab to remain open.
|
||
</p>
|
||
</div>
|
||
|
||
<Button
|
||
onClick={startScrape}
|
||
disabled={!urlValid || !vayneConfigured || stage === "scraping"}
|
||
className="bg-gradient-to-r from-blue-500 to-purple-600 hover:from-blue-600 hover:to-purple-700 text-white font-medium px-8 shadow-lg hover:shadow-blue-500/25 transition-all disabled:opacity-50"
|
||
>
|
||
Start LinkedIn Scrape
|
||
</Button>
|
||
</Card>
|
||
|
||
{/* Scrape progress */}
|
||
{(stage === "scraping") && (
|
||
<ProgressCard
|
||
title="Scraping LinkedIn profiles via Vayne..."
|
||
current={scrapeProgress.current}
|
||
total={scrapeProgress.total}
|
||
subtitle="Creating order → Scraping → Generating export..."
|
||
status="running"
|
||
/>
|
||
)}
|
||
|
||
{/* Results table */}
|
||
{results.length > 0 && (
|
||
<Card className="bg-[#111118] border-[#1e1e2e] p-6 space-y-4">
|
||
<h2 className="text-base font-semibold text-white flex items-center gap-2">
|
||
<span className="w-6 h-6 rounded-full bg-blue-500/20 text-blue-400 text-xs flex items-center justify-center font-bold">2</span>
|
||
Scraped Profiles ({results.length})
|
||
</h2>
|
||
<p className="text-xs text-gray-500">Select profiles to include in email enrichment</p>
|
||
<ResultsTable
|
||
rows={results}
|
||
selectable
|
||
onSelectionChange={setSelectedIds}
|
||
/>
|
||
</Card>
|
||
)}
|
||
|
||
{/* Step 3: Enrich */}
|
||
{(stage === "scraped" || stage === "enriching" || stage === "done") && results.length > 0 && (
|
||
<Card className="bg-[#111118] border-[#1e1e2e] p-6 space-y-5">
|
||
<h2 className="text-base font-semibold text-white flex items-center gap-2">
|
||
<span className="w-6 h-6 rounded-full bg-blue-500/20 text-blue-400 text-xs flex items-center justify-center font-bold">3</span>
|
||
Enrich with Emails
|
||
</h2>
|
||
|
||
<div className="flex flex-wrap gap-2">
|
||
{CATEGORY_OPTIONS.map(opt => (
|
||
<button
|
||
key={opt.value}
|
||
onClick={() => setCategories(prev =>
|
||
prev.includes(opt.value) ? prev.filter(c => c !== opt.value) : [...prev, opt.value]
|
||
)}
|
||
className={`px-3 py-1.5 rounded-lg text-sm font-medium border transition-all ${
|
||
categories.includes(opt.value)
|
||
? "bg-blue-500/20 text-blue-300 border-blue-500/40"
|
||
: "bg-[#0d0d18] text-gray-400 border-[#2e2e3e] hover:border-blue-500/30"
|
||
}`}
|
||
>
|
||
{opt.label}
|
||
</button>
|
||
))}
|
||
</div>
|
||
|
||
{stage === "enriching" && (
|
||
<ProgressCard
|
||
title="Enriching profiles..."
|
||
current={enrichProgress.current}
|
||
total={enrichProgress.total}
|
||
subtitle="Finding emails via Anymailfinder"
|
||
status="running"
|
||
/>
|
||
)}
|
||
|
||
{stage === "scraped" && (
|
||
<Button
|
||
onClick={startEnrich}
|
||
disabled={!selectedIds.length || !categories.length}
|
||
className="bg-gradient-to-r from-blue-500 to-purple-600 hover:from-blue-600 hover:to-purple-700 text-white font-medium px-8 shadow-lg hover:shadow-blue-500/25 transition-all"
|
||
>
|
||
Enrich {selectedIds.length} Selected Profiles with Emails
|
||
</Button>
|
||
)}
|
||
</Card>
|
||
)}
|
||
|
||
{/* Export */}
|
||
{(stage === "done" || emailsFound > 0) && (
|
||
<Card className="bg-[#111118] border-[#1e1e2e] p-6">
|
||
<ExportButtons
|
||
rows={exportRows}
|
||
filename={`linkedin-leads-${scrapeJobId?.slice(0, 8) || "export"}`}
|
||
summary={`${emailsFound} emails found from ${results.length} profiles`}
|
||
/>
|
||
</Card>
|
||
)}
|
||
|
||
{/* Empty state */}
|
||
{stage === "idle" && (
|
||
<EmptyState
|
||
icon={Linkedin}
|
||
title="Start by pasting a Sales Navigator URL"
|
||
description="Configure your search filters in Sales Navigator, copy the URL, and paste it above to begin scraping."
|
||
/>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|