Full-stack Next.js 16 app with three scraping pipelines: - AirScale CSV → Anymailfinder Bulk Decision Maker search - LinkedIn Sales Navigator → Vayne → Anymailfinder email enrichment - Apify Google SERP → domain extraction → Anymailfinder bulk enrichment Includes Docker multi-stage build + docker-compose for Coolify deployment. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
330 lines
13 KiB
TypeScript
330 lines
13 KiB
TypeScript
"use client";
|
|
|
|
import { useState } from "react";
|
|
import { Card } from "@/components/ui/card";
|
|
import { Button } from "@/components/ui/button";
|
|
import { Input } from "@/components/ui/input";
|
|
import { Label } from "@/components/ui/label";
|
|
import { ProgressCard } from "@/components/shared/ProgressCard";
|
|
import { ResultsTable, type ResultRow } from "@/components/shared/ResultsTable";
|
|
import { ExportButtons } from "@/components/shared/ExportButtons";
|
|
import { EmptyState } from "@/components/shared/EmptyState";
|
|
import { toast } from "sonner";
|
|
import { Search, ChevronRight } from "lucide-react";
|
|
import { useAppStore } from "@/lib/store";
|
|
import type { DecisionMakerCategory } from "@/lib/services/anymailfinder";
|
|
import type { ExportRow } from "@/lib/utils/csv";
|
|
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
|
import { Checkbox } from "@/components/ui/checkbox";
|
|
|
|
const RESULT_OPTIONS = [10, 25, 50, 100, 200];
|
|
|
|
const CATEGORY_OPTIONS: { value: DecisionMakerCategory; label: string }[] = [
|
|
{ value: "ceo", label: "CEO / Owner / Founder" },
|
|
{ value: "engineering", label: "Engineering" },
|
|
{ value: "marketing", label: "Marketing" },
|
|
{ value: "sales", label: "Sales" },
|
|
{ value: "operations", label: "Operations" },
|
|
{ value: "finance", label: "Finance" },
|
|
{ value: "hr", label: "HR" },
|
|
{ value: "it", label: "IT" },
|
|
{ value: "buyer", label: "Procurement" },
|
|
{ value: "logistics", label: "Logistics" },
|
|
];
|
|
|
|
type Stage = "idle" | "running" | "done" | "failed";
|
|
|
|
export default function SerpPage() {
|
|
const [query, setQuery] = useState("");
|
|
const [numResults, setNumResults] = useState(50);
|
|
const [country, setCountry] = useState("de");
|
|
const [language, setLanguage] = useState("de");
|
|
const [filterSocial, setFilterSocial] = useState(true);
|
|
const [categories, setCategories] = useState<DecisionMakerCategory[]>(["ceo"]);
|
|
const [stage, setStage] = useState<Stage>("idle");
|
|
const [jobId, setJobId] = useState<string | null>(null);
|
|
const [progress, setProgress] = useState({ current: 0, total: 0, phase: "" });
|
|
const [results, setResults] = useState<ResultRow[]>([]);
|
|
const { addJob, updateJob, removeJob } = useAppStore();
|
|
|
|
// maxPages: since Google limits to 10 results/page, divide by 10
|
|
const maxPages = Math.max(1, Math.ceil(numResults / 10));
|
|
|
|
const startJob = async () => {
|
|
if (!query.trim()) return toast.error("Enter a search query");
|
|
if (!categories.length) return toast.error("Select at least one category");
|
|
|
|
setStage("running");
|
|
setResults([]);
|
|
setProgress({ current: 0, total: numResults, phase: "Searching Google..." });
|
|
|
|
try {
|
|
const res = await fetch("/api/jobs/serp-enrich", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({
|
|
query,
|
|
maxPages,
|
|
countryCode: country,
|
|
languageCode: language,
|
|
filterSocial,
|
|
categories,
|
|
}),
|
|
});
|
|
const data = await res.json() as { jobId?: string; error?: string };
|
|
if (!res.ok || !data.jobId) throw new Error(data.error || "Failed");
|
|
|
|
setJobId(data.jobId);
|
|
addJob({ id: data.jobId, type: "serp", status: "running", progress: 0, total: numResults });
|
|
pollJob(data.jobId);
|
|
} catch (err) {
|
|
toast.error(err instanceof Error ? err.message : "Failed to start");
|
|
setStage("failed");
|
|
}
|
|
};
|
|
|
|
const pollJob = (id: string) => {
|
|
let phase = "Searching Google...";
|
|
const interval = setInterval(async () => {
|
|
try {
|
|
const res = await fetch(`/api/jobs/${id}/status`);
|
|
const data = await res.json() as {
|
|
status: string; totalLeads: number; emailsFound: number;
|
|
results: ResultRow[];
|
|
};
|
|
|
|
// Infer phase from progress
|
|
if (data.totalLeads > 0 && data.emailsFound === 0) phase = "Enriching domains with Anymailfinder...";
|
|
if (data.emailsFound > 0) phase = `Found ${data.emailsFound} emails so far...`;
|
|
|
|
setProgress({ current: data.emailsFound, total: data.totalLeads || numResults, phase });
|
|
if (data.results?.length) setResults(data.results);
|
|
|
|
updateJob(id, { status: data.status, progress: data.emailsFound, total: data.totalLeads });
|
|
|
|
if (data.status === "complete" || data.status === "failed") {
|
|
clearInterval(interval);
|
|
removeJob(id);
|
|
setResults(data.results || []);
|
|
if (data.status === "complete") {
|
|
setStage("done");
|
|
toast.success(`Done! Found ${data.emailsFound} emails`);
|
|
} else {
|
|
setStage("failed");
|
|
toast.error("Job failed. Check your API keys in Settings.");
|
|
}
|
|
}
|
|
} catch {
|
|
clearInterval(interval);
|
|
setStage("failed");
|
|
}
|
|
}, 2000);
|
|
};
|
|
|
|
const exportRows: ExportRow[] = results.map(r => ({
|
|
company_name: r.companyName,
|
|
domain: r.domain,
|
|
contact_name: r.contactName,
|
|
contact_title: r.contactTitle,
|
|
email: r.email,
|
|
confidence_score: r.confidence !== undefined ? Math.round(r.confidence * 100) : undefined,
|
|
source_tab: "serp",
|
|
job_id: jobId || "",
|
|
found_at: new Date().toISOString(),
|
|
}));
|
|
|
|
const emailsFound = results.filter(r => r.email).length;
|
|
const hitRate = results.length > 0 ? Math.round((emailsFound / results.length) * 100) : 0;
|
|
|
|
return (
|
|
<div className="space-y-6 max-w-5xl">
|
|
{/* Header */}
|
|
<div className="relative rounded-2xl bg-gradient-to-r from-purple-600/10 to-blue-600/10 border border-[#1e1e2e] p-6 overflow-hidden">
|
|
<div className="absolute inset-0 bg-gradient-to-br from-purple-500/5 to-transparent" />
|
|
<div className="relative">
|
|
<div className="flex items-center gap-2 text-sm text-purple-400 mb-2">
|
|
<Search className="w-4 h-4" />
|
|
<span>Tab 3</span>
|
|
<ChevronRight className="w-3 h-3" />
|
|
<span>Google SERP</span>
|
|
</div>
|
|
<h1 className="text-2xl font-bold text-white">SERP → Email Enrichment</h1>
|
|
<p className="text-gray-400 mt-1 text-sm">
|
|
Scrape Google search results via Apify, extract domains, then find decision maker emails.
|
|
</p>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Step 1: Configure */}
|
|
<Card className="bg-[#111118] border-[#1e1e2e] p-6 space-y-5">
|
|
<h2 className="text-base font-semibold text-white flex items-center gap-2">
|
|
<span className="w-6 h-6 rounded-full bg-purple-500/20 text-purple-400 text-xs flex items-center justify-center font-bold">1</span>
|
|
Search Configuration
|
|
</h2>
|
|
|
|
<div>
|
|
<Label className="text-gray-300 text-sm mb-1.5 block">Search Term</Label>
|
|
<Input
|
|
placeholder='e.g. "Solaranlage Installateur Deutschland"'
|
|
value={query}
|
|
onChange={e => setQuery(e.target.value)}
|
|
className="bg-[#0d0d18] border-[#2e2e3e] text-white placeholder:text-gray-600 focus:border-blue-500"
|
|
/>
|
|
</div>
|
|
|
|
<div className="grid grid-cols-3 gap-4">
|
|
<div>
|
|
<Label className="text-gray-300 text-sm mb-1.5 block">Number of Results</Label>
|
|
<Select value={String(numResults)} onValueChange={v => setNumResults(Number(v))}>
|
|
<SelectTrigger className="bg-[#0d0d18] border-[#2e2e3e] text-white">
|
|
<SelectValue />
|
|
</SelectTrigger>
|
|
<SelectContent className="bg-[#111118] border-[#2e2e3e]">
|
|
{RESULT_OPTIONS.map(n => (
|
|
<SelectItem key={n} value={String(n)} className="text-gray-300">{n} results</SelectItem>
|
|
))}
|
|
</SelectContent>
|
|
</Select>
|
|
<p className="text-xs text-gray-500 mt-1">~{maxPages} page{maxPages > 1 ? "s" : ""} of Google</p>
|
|
</div>
|
|
<div>
|
|
<Label className="text-gray-300 text-sm mb-1.5 block">Country</Label>
|
|
<Select value={country} onValueChange={v => setCountry(v ?? "de")}>
|
|
<SelectTrigger className="bg-[#0d0d18] border-[#2e2e3e] text-white">
|
|
<SelectValue />
|
|
</SelectTrigger>
|
|
<SelectContent className="bg-[#111118] border-[#2e2e3e]">
|
|
{[
|
|
{ value: "de", label: "Germany (DE)" },
|
|
{ value: "at", label: "Austria (AT)" },
|
|
{ value: "ch", label: "Switzerland (CH)" },
|
|
{ value: "us", label: "United States (US)" },
|
|
{ value: "gb", label: "United Kingdom (GB)" },
|
|
].map(c => (
|
|
<SelectItem key={c.value} value={c.value} className="text-gray-300">{c.label}</SelectItem>
|
|
))}
|
|
</SelectContent>
|
|
</Select>
|
|
</div>
|
|
<div>
|
|
<Label className="text-gray-300 text-sm mb-1.5 block">Language</Label>
|
|
<Select value={language} onValueChange={v => setLanguage(v ?? "de")}>
|
|
<SelectTrigger className="bg-[#0d0d18] border-[#2e2e3e] text-white">
|
|
<SelectValue />
|
|
</SelectTrigger>
|
|
<SelectContent className="bg-[#111118] border-[#2e2e3e]">
|
|
{[
|
|
{ value: "de", label: "German" },
|
|
{ value: "en", label: "English" },
|
|
{ value: "fr", label: "French" },
|
|
].map(l => (
|
|
<SelectItem key={l.value} value={l.value} className="text-gray-300">{l.label}</SelectItem>
|
|
))}
|
|
</SelectContent>
|
|
</Select>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="flex items-start gap-3">
|
|
<Checkbox
|
|
id="filterSocial"
|
|
checked={filterSocial}
|
|
onCheckedChange={v => setFilterSocial(!!v)}
|
|
className="border-[#2e2e3e] mt-0.5"
|
|
/>
|
|
<div>
|
|
<label htmlFor="filterSocial" className="text-sm text-gray-300 cursor-pointer">
|
|
Exclude social media, directories & aggregators
|
|
</label>
|
|
<p className="text-xs text-gray-500 mt-0.5">
|
|
Filters out: LinkedIn, Facebook, Instagram, Yelp, Wikipedia, Xing, Twitter/X, YouTube, Google Maps
|
|
</p>
|
|
</div>
|
|
</div>
|
|
</Card>
|
|
|
|
{/* Step 2: Categories */}
|
|
<Card className="bg-[#111118] border-[#1e1e2e] p-6 space-y-4">
|
|
<h2 className="text-base font-semibold text-white flex items-center gap-2">
|
|
<span className="w-6 h-6 rounded-full bg-purple-500/20 text-purple-400 text-xs flex items-center justify-center font-bold">2</span>
|
|
Decision Maker Categories
|
|
</h2>
|
|
<div className="flex flex-wrap gap-2">
|
|
{CATEGORY_OPTIONS.map(opt => (
|
|
<button
|
|
key={opt.value}
|
|
onClick={() => setCategories(prev =>
|
|
prev.includes(opt.value) ? prev.filter(c => c !== opt.value) : [...prev, opt.value]
|
|
)}
|
|
className={`px-3 py-1.5 rounded-lg text-sm font-medium border transition-all ${
|
|
categories.includes(opt.value)
|
|
? "bg-purple-500/20 text-purple-300 border-purple-500/40"
|
|
: "bg-[#0d0d18] text-gray-400 border-[#2e2e3e] hover:border-purple-500/30"
|
|
}`}
|
|
>
|
|
{opt.label}
|
|
</button>
|
|
))}
|
|
</div>
|
|
</Card>
|
|
|
|
{/* Run */}
|
|
<Card className="bg-[#111118] border-[#1e1e2e] p-6 space-y-4">
|
|
<h2 className="text-base font-semibold text-white flex items-center gap-2">
|
|
<span className="w-6 h-6 rounded-full bg-purple-500/20 text-purple-400 text-xs flex items-center justify-center font-bold">3</span>
|
|
Run Pipeline
|
|
</h2>
|
|
|
|
{stage === "idle" || stage === "failed" ? (
|
|
<Button
|
|
onClick={startJob}
|
|
disabled={!query.trim() || !categories.length}
|
|
className="bg-gradient-to-r from-purple-500 to-blue-600 hover:from-purple-600 hover:to-blue-700 text-white font-medium px-8 shadow-lg hover:shadow-purple-500/25 transition-all"
|
|
>
|
|
Start SERP Search
|
|
</Button>
|
|
) : stage === "running" ? (
|
|
<ProgressCard
|
|
title="Running SERP pipeline..."
|
|
current={progress.current}
|
|
total={progress.total}
|
|
subtitle={progress.phase}
|
|
status="running"
|
|
/>
|
|
) : (
|
|
<ProgressCard
|
|
title="Pipeline complete"
|
|
current={emailsFound}
|
|
total={results.length}
|
|
subtitle={`Hit rate: ${hitRate}% · ${results.length} domains scraped`}
|
|
status="complete"
|
|
/>
|
|
)}
|
|
</Card>
|
|
|
|
{/* Results */}
|
|
{results.length > 0 && (
|
|
<Card className="bg-[#111118] border-[#1e1e2e] p-6 space-y-4">
|
|
<div className="flex items-center justify-between flex-wrap gap-3">
|
|
<h2 className="text-base font-semibold text-white">Results</h2>
|
|
<ExportButtons
|
|
rows={exportRows}
|
|
filename={`serp-leads-${jobId?.slice(0, 8) || "export"}`}
|
|
summary={`${emailsFound} emails found • ${hitRate}% hit rate`}
|
|
/>
|
|
</div>
|
|
<ResultsTable rows={results} loading={stage === "running" && results.length === 0} />
|
|
</Card>
|
|
)}
|
|
|
|
{stage === "idle" && (
|
|
<EmptyState
|
|
icon={Search}
|
|
title="Configure and run a SERP search"
|
|
description="Enter a search term, select your target categories, and hit Start to find leads from Google search results."
|
|
/>
|
|
)}
|
|
</div>
|
|
);
|
|
}
|