Initial commit: LeadFlow lead generation platform
Full-stack Next.js 16 app with three scraping pipelines: - AirScale CSV → Anymailfinder Bulk Decision Maker search - LinkedIn Sales Navigator → Vayne → Anymailfinder email enrichment - Apify Google SERP → domain extraction → Anymailfinder bulk enrichment Includes Docker multi-stage build + docker-compose for Coolify deployment. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
155
app/api/jobs/serp-enrich/route.ts
Normal file
155
app/api/jobs/serp-enrich/route.ts
Normal file
@@ -0,0 +1,155 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { decrypt } from "@/lib/utils/encryption";
|
||||
import { isSocialOrDirectory } from "@/lib/utils/domains";
|
||||
import { runGoogleSerpScraper, pollRunStatus, fetchDatasetItems } from "@/lib/services/apify";
|
||||
import { bulkSearchDomains, type DecisionMakerCategory } from "@/lib/services/anymailfinder";
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const body = await req.json() as {
|
||||
query: string;
|
||||
maxPages: number;
|
||||
countryCode: string;
|
||||
languageCode: string;
|
||||
filterSocial: boolean;
|
||||
categories: DecisionMakerCategory[];
|
||||
selectedDomains?: string[];
|
||||
};
|
||||
|
||||
const apifyCred = await prisma.apiCredential.findUnique({ where: { service: "apify" } });
|
||||
const anymailCred = await prisma.apiCredential.findUnique({ where: { service: "anymailfinder" } });
|
||||
|
||||
if (!apifyCred?.value) return NextResponse.json({ error: "Apify API token not configured" }, { status: 400 });
|
||||
if (!anymailCred?.value) return NextResponse.json({ error: "Anymailfinder API key not configured" }, { status: 400 });
|
||||
|
||||
const apifyToken = decrypt(apifyCred.value);
|
||||
const anymailKey = decrypt(anymailCred.value);
|
||||
|
||||
const job = await prisma.job.create({
|
||||
data: {
|
||||
type: "serp",
|
||||
status: "running",
|
||||
config: JSON.stringify(body),
|
||||
totalLeads: 0,
|
||||
},
|
||||
});
|
||||
|
||||
runSerpEnrich(job.id, body, apifyToken, anymailKey).catch(console.error);
|
||||
|
||||
return NextResponse.json({ jobId: job.id });
|
||||
} catch (err) {
|
||||
console.error("POST /api/jobs/serp-enrich error:", err);
|
||||
return NextResponse.json({ error: "Failed to start job" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
async function runSerpEnrich(
|
||||
jobId: string,
|
||||
params: {
|
||||
query: string; maxPages: number; countryCode: string; languageCode: string;
|
||||
filterSocial: boolean; categories: DecisionMakerCategory[]; selectedDomains?: string[];
|
||||
},
|
||||
apifyToken: string,
|
||||
anymailKey: string
|
||||
) {
|
||||
try {
|
||||
// 1. Run Apify SERP scraper
|
||||
const runId = await runGoogleSerpScraper(
|
||||
params.query, params.maxPages, params.countryCode, params.languageCode, apifyToken
|
||||
);
|
||||
|
||||
// 2. Poll until complete
|
||||
let runStatus = "";
|
||||
let datasetId = "";
|
||||
while (runStatus !== "SUCCEEDED" && runStatus !== "FAILED" && runStatus !== "ABORTED") {
|
||||
await sleep(3000);
|
||||
const result = await pollRunStatus(runId, apifyToken);
|
||||
runStatus = result.status;
|
||||
datasetId = result.defaultDatasetId;
|
||||
}
|
||||
|
||||
if (runStatus !== "SUCCEEDED") throw new Error(`Apify run ${runStatus}`);
|
||||
|
||||
// 3. Fetch results
|
||||
let serpResults = await fetchDatasetItems(datasetId, apifyToken);
|
||||
|
||||
// 4. Filter social/directories
|
||||
if (params.filterSocial) {
|
||||
serpResults = serpResults.filter(r => !isSocialOrDirectory(r.domain));
|
||||
}
|
||||
|
||||
// 5. Deduplicate domains
|
||||
const seenDomains = new Set<string>();
|
||||
const uniqueResults = serpResults.filter(r => {
|
||||
if (!r.domain || seenDomains.has(r.domain)) return false;
|
||||
seenDomains.add(r.domain);
|
||||
return true;
|
||||
});
|
||||
|
||||
// 6. Apply selectedDomains filter if provided
|
||||
const filteredResults = params.selectedDomains?.length
|
||||
? uniqueResults.filter(r => params.selectedDomains!.includes(r.domain))
|
||||
: uniqueResults;
|
||||
|
||||
const domains = filteredResults.map(r => r.domain);
|
||||
const serpMap = new Map(filteredResults.map(r => [r.domain, r]));
|
||||
|
||||
await prisma.job.update({
|
||||
where: { id: jobId },
|
||||
data: { totalLeads: domains.length },
|
||||
});
|
||||
|
||||
// 7. Enrich with Anymailfinder Bulk API
|
||||
const enrichResults = await bulkSearchDomains(
|
||||
domains,
|
||||
params.categories,
|
||||
anymailKey,
|
||||
async (_completed, total) => {
|
||||
await prisma.job.update({ where: { id: jobId }, data: { totalLeads: total } });
|
||||
}
|
||||
);
|
||||
|
||||
// 8. Store results
|
||||
let emailsFound = 0;
|
||||
for (const result of enrichResults) {
|
||||
const serpData = serpMap.get(result.domain || "");
|
||||
const hasEmail = !!result.valid_email;
|
||||
if (hasEmail) emailsFound++;
|
||||
|
||||
await prisma.leadResult.create({
|
||||
data: {
|
||||
jobId,
|
||||
companyName: serpData?.title || null,
|
||||
domain: result.domain || null,
|
||||
contactName: result.person_full_name || null,
|
||||
contactTitle: result.person_job_title || null,
|
||||
email: result.email || null,
|
||||
confidence: result.valid_email ? 1.0 : result.email_status === "risky" ? 0.5 : 0,
|
||||
linkedinUrl: result.person_linkedin_url || null,
|
||||
source: JSON.stringify({
|
||||
url: serpData?.url,
|
||||
description: serpData?.description,
|
||||
position: serpData?.position,
|
||||
email_status: result.email_status,
|
||||
}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await prisma.job.update({
|
||||
where: { id: jobId },
|
||||
data: { status: "complete", emailsFound, totalLeads: enrichResults.length },
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
await prisma.job.update({
|
||||
where: { id: jobId },
|
||||
data: { status: "failed", error: message },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
Reference in New Issue
Block a user