Initial commit: LeadFlow lead generation platform
Full-stack Next.js 16 app with three scraping pipelines: - AirScale CSV → Anymailfinder Bulk Decision Maker search - LinkedIn Sales Navigator → Vayne → Anymailfinder email enrichment - Apify Google SERP → domain extraction → Anymailfinder bulk enrichment Includes Docker multi-stage build + docker-compose for Coolify deployment. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
116
app/api/jobs/vayne-scrape/route.ts
Normal file
116
app/api/jobs/vayne-scrape/route.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { decrypt } from "@/lib/utils/encryption";
|
||||
import { createOrder, getOrderStatus, triggerExport, downloadOrderCSV } from "@/lib/services/vayne";
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const body = await req.json() as { salesNavUrl: string; maxResults: number };
|
||||
const { salesNavUrl, maxResults } = body;
|
||||
|
||||
if (!salesNavUrl?.includes("linkedin.com/sales")) {
|
||||
return NextResponse.json({ error: "Invalid Sales Navigator URL" }, { status: 400 });
|
||||
}
|
||||
|
||||
const cred = await prisma.apiCredential.findUnique({ where: { service: "vayne" } });
|
||||
if (!cred?.value) {
|
||||
return NextResponse.json({ error: "Vayne API token not configured" }, { status: 400 });
|
||||
}
|
||||
const apiToken = decrypt(cred.value);
|
||||
|
||||
const job = await prisma.job.create({
|
||||
data: {
|
||||
type: "linkedin",
|
||||
status: "running",
|
||||
config: JSON.stringify({ salesNavUrl, maxResults }),
|
||||
totalLeads: 0,
|
||||
},
|
||||
});
|
||||
|
||||
runVayneScrape(job.id, salesNavUrl, maxResults, apiToken).catch(console.error);
|
||||
|
||||
return NextResponse.json({ jobId: job.id });
|
||||
} catch (err) {
|
||||
console.error("POST /api/jobs/vayne-scrape error:", err);
|
||||
return NextResponse.json({ error: "Failed to start scrape" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
async function runVayneScrape(
|
||||
jobId: string,
|
||||
salesNavUrl: string,
|
||||
maxResults: number,
|
||||
apiToken: string
|
||||
) {
|
||||
try {
|
||||
// 1. Create Vayne order
|
||||
const order = await createOrder(salesNavUrl, maxResults, apiToken, `LeadFlow-${jobId.slice(0, 8)}`);
|
||||
const orderId = order.id;
|
||||
|
||||
await prisma.job.update({
|
||||
where: { id: jobId },
|
||||
data: { config: JSON.stringify({ salesNavUrl, maxResults, vayneOrderId: orderId }) },
|
||||
});
|
||||
|
||||
// 2. Poll until finished
|
||||
let status = order.scraping_status;
|
||||
let scraped = 0;
|
||||
while (status !== "finished" && status !== "failed") {
|
||||
await sleep(5000);
|
||||
const updated = await getOrderStatus(orderId, apiToken);
|
||||
status = updated.scraping_status;
|
||||
scraped = updated.scraped || 0;
|
||||
await prisma.job.update({ where: { id: jobId }, data: { totalLeads: scraped } });
|
||||
}
|
||||
|
||||
if (status === "failed") {
|
||||
throw new Error("Vayne scraping failed");
|
||||
}
|
||||
|
||||
// 3. Trigger export
|
||||
let exportOrder = await triggerExport(orderId, apiToken);
|
||||
|
||||
// 4. Poll for export completion
|
||||
let exportStatus = exportOrder.exports?.[0]?.status;
|
||||
while (exportStatus !== "completed") {
|
||||
await sleep(3000);
|
||||
exportOrder = await getOrderStatus(orderId, apiToken);
|
||||
exportStatus = exportOrder.exports?.[0]?.status;
|
||||
if (exportStatus === undefined) break; // fallback
|
||||
}
|
||||
|
||||
const fileUrl = exportOrder.exports?.[0]?.file_url;
|
||||
if (!fileUrl) throw new Error("No export file URL returned by Vayne");
|
||||
|
||||
// 5. Download and parse CSV
|
||||
const profiles = await downloadOrderCSV(fileUrl);
|
||||
|
||||
// 6. Store results
|
||||
await prisma.leadResult.createMany({
|
||||
data: profiles.map(p => ({
|
||||
jobId,
|
||||
companyName: p.company || null,
|
||||
domain: p.companyDomain || null,
|
||||
contactName: p.fullName || null,
|
||||
contactTitle: p.title || null,
|
||||
linkedinUrl: p.linkedinUrl || null,
|
||||
source: JSON.stringify({ location: p.location }),
|
||||
})),
|
||||
});
|
||||
|
||||
await prisma.job.update({
|
||||
where: { id: jobId },
|
||||
data: { status: "complete", totalLeads: profiles.length },
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
await prisma.job.update({
|
||||
where: { id: jobId },
|
||||
data: { status: "failed", error: message },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise(r => setTimeout(r, ms));
|
||||
}
|
||||
Reference in New Issue
Block a user