feat: Apify Meta reklam ingest akisini ekle
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
import { getApiAdmin, requestAuditContext } from "@/lib/admin-api";
|
||||
import { importApifyAds, runApifyActor } from "@/lib/apify";
|
||||
import { prisma } from "@/lib/db";
|
||||
|
||||
const schema = z.object({
|
||||
searchTerms: z.array(z.string().trim().min(2).max(100)).min(1).max(10),
|
||||
country: z.string().trim().toUpperCase().refine((value) => value === "ALL" || /^[A-Z]{2}$/.test(value)),
|
||||
adActiveStatus: z.enum(["ACTIVE", "INACTIVE", "ALL"]).default("ACTIVE"),
|
||||
mediaType: z.enum(["ALL", "IMAGE", "VIDEO", "MEME", "NONE"]).default("ALL"),
|
||||
maxResults: z.coerce.number().int().min(1).max(1000).default(100),
|
||||
scrapeAdDetails: z.boolean().default(true),
|
||||
includeAboutPage: z.boolean().default(false),
|
||||
maxCostUsd: z.coerce.number().min(0.1).max(10).default(1)
|
||||
});
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const actor = await getApiAdmin();
|
||||
if (!actor) return NextResponse.json({ error: "FORBIDDEN" }, { status: 403 });
|
||||
const parsed = schema.safeParse(await request.json().catch(() => null));
|
||||
if (!parsed.success) return NextResponse.json({ error: "INVALID_INPUT" }, { status: 400 });
|
||||
const auditContext = requestAuditContext(request);
|
||||
const job = await prisma.ingestJob.create({
|
||||
data: { source: "apify", type: "meta-ads-library", status: "RUNNING", startedAt: new Date(), metadata: { searchTerms: parsed.data.searchTerms, country: parsed.data.country, maxResults: parsed.data.maxResults } }
|
||||
});
|
||||
try {
|
||||
const records = await runApifyActor(parsed.data);
|
||||
const result = await importApifyAds(records);
|
||||
await prisma.$transaction([
|
||||
prisma.ingestJob.update({ where: { id: job.id }, data: { status: "COMPLETED", finishedAt: new Date(), recordsImported: result.imported, recordsFailed: result.failed, metadata: { searchTerms: parsed.data.searchTerms, country: parsed.data.country, received: records.length } } }),
|
||||
prisma.adminAuditLog.create({ data: { actorId: actor.id, actorEmail: actor.email, action: "APIFY_INGEST_COMPLETED", targetType: "IngestJob", targetId: job.id, summary: `${result.imported} Meta reklamı Apify üzerinden işlendi.`, details: { ...result, received: records.length, searchTerms: parsed.data.searchTerms, country: parsed.data.country }, ...auditContext } })
|
||||
]);
|
||||
return NextResponse.json({ ok: true, jobId: job.id, ...result, received: records.length });
|
||||
} catch (error) {
|
||||
const code = error instanceof Error && /^APIFY_[A-Z0-9_]+$/.test(error.message) ? error.message : "APIFY_INGEST_FAILED";
|
||||
await prisma.$transaction([
|
||||
prisma.ingestJob.update({ where: { id: job.id }, data: { status: "FAILED", finishedAt: new Date(), errorMessage: code } }),
|
||||
prisma.adminAuditLog.create({ data: { actorId: actor.id, actorEmail: actor.email, action: "APIFY_INGEST_FAILED", targetType: "IngestJob", targetId: job.id, summary: `Apify ingest başarısız: ${code}`, details: { searchTerms: parsed.data.searchTerms, country: parsed.data.country }, ...auditContext } })
|
||||
]);
|
||||
return NextResponse.json({ error: code }, { status: code === "APIFY_NOT_CONFIGURED" ? 503 : 502 });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,9 +2,10 @@ import { requireAdmin } from "@/lib/auth/current-user";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { Card } from "@/components/ui/card";
|
||||
import { AdminNav } from "@/components/admin/admin-nav";
|
||||
import { ApifyIngestForm } from "@/components/admin/apify-ingest-form";
|
||||
|
||||
export default async function AdminDataPage() {
|
||||
await requireAdmin();
|
||||
const [ads, stores, users, jobs] = await Promise.all([prisma.ad.count(), prisma.store.count(), prisma.user.count(), prisma.ingestJob.findMany({ orderBy: { createdAt: "desc" }, take: 10 })]);
|
||||
return <div><h1 className="mb-2 text-3xl font-black">Veri İşleri</h1><p className="mb-6 text-slate-500">Reklam, mağaza ve ingest operasyonları.</p><AdminNav /><div className="grid gap-4 md:grid-cols-3"><Card><b>{ads}</b><br />Ads</Card><Card><b>{stores}</b><br />Stores</Card><Card><b>{users}</b><br />Users</Card></div><Card className="mt-5"><h2 className="mb-3 font-black">Ingest Jobs</h2>{jobs.map((j) => <div key={j.id} className="border-t py-2 text-sm">{j.source} · {j.type} · {j.status} · {j.recordsImported} records</div>)}</Card></div>;
|
||||
return <div><h1 className="mb-2 text-3xl font-black">Veri İşleri</h1><p className="mb-6 text-slate-500">Reklam, mağaza ve ingest operasyonları.</p><AdminNav /><div className="grid gap-4 md:grid-cols-3"><Card><b className="text-3xl">{ads}</b><br />Ads</Card><Card><b className="text-3xl">{stores}</b><br />Stores</Card><Card><b className="text-3xl">{users}</b><br />Users</Card></div><Card className="mt-5"><div className="mb-4"><h2 className="text-xl font-black">Apify · Meta Ads Library</h2><p className="mt-1 text-sm text-slate-500">Anahtar kelimeyle gerçek Meta reklamlarını çekin. Her çalışmada maliyet üst sınırı uygulanır ve tekrar eden reklamlar güncellenir.</p></div><ApifyIngestForm configured={Boolean(process.env.APIFY_TOKEN)} /></Card><Card className="mt-5"><h2 className="mb-3 font-black">Ingest Jobs</h2>{jobs.map((j) => <div key={j.id} className="border-t py-3 text-sm"><b>{j.source} · {j.type}</b><div className="text-slate-500">{j.status} · {j.recordsImported} başarılı · {j.recordsFailed} hatalı</div>{j.errorMessage && <div className="text-rose-600">{j.errorMessage}</div>}</div>)}{!jobs.length && <p className="text-sm text-slate-500">Henüz ingest işi yok.</p>}</Card></div>;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"use client";
|
||||
|
||||
import { FormEvent, useState } from "react";
|
||||
import { useRouter } from "next/navigation";
|
||||
|
||||
export function ApifyIngestForm({ configured }: { configured: boolean }) {
|
||||
const router = useRouter();
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [message, setMessage] = useState("");
|
||||
const [error, setError] = useState(false);
|
||||
|
||||
async function submit(event: FormEvent<HTMLFormElement>) {
|
||||
event.preventDefault();
|
||||
const form = new FormData(event.currentTarget);
|
||||
const searchTerms = String(form.get("searchTerms") || "").split(",").map((term) => term.trim()).filter(Boolean);
|
||||
const maxResults = Number(form.get("maxResults"));
|
||||
const maxCostUsd = Number(form.get("maxCostUsd"));
|
||||
if (!window.confirm(`${searchTerms.join(", ")} için en fazla ${maxResults} reklam çekilsin mi? Apify harcama üst sınırı $${maxCostUsd.toFixed(2)}.`)) return;
|
||||
setBusy(true); setMessage("");
|
||||
try {
|
||||
const response = await fetch("/api/admin/ingest/apify", {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ searchTerms, country: form.get("country"), adActiveStatus: form.get("adActiveStatus"), mediaType: form.get("mediaType"), maxResults, maxCostUsd, scrapeAdDetails: form.get("scrapeAdDetails") === "on", includeAboutPage: form.get("includeAboutPage") === "on" })
|
||||
});
|
||||
const data = await response.json().catch(() => ({}));
|
||||
if (!response.ok) throw new Error(data.error || "APIFY_INGEST_FAILED");
|
||||
setError(false); setMessage(`${data.imported} reklam işlendi, ${data.failed} kayıt atlandı.`); router.refresh();
|
||||
} catch (caught) {
|
||||
setError(true); setMessage(caught instanceof Error ? caught.message : "İşlem başarısız");
|
||||
} finally { setBusy(false); }
|
||||
}
|
||||
|
||||
return (
|
||||
<form onSubmit={submit} className="grid gap-3 lg:grid-cols-2">
|
||||
<input name="searchTerms" required minLength={2} maxLength={500} placeholder="Anahtar kelimeler: skincare, dog collar" className="rounded-xl border border-slate-200 px-3 py-3 lg:col-span-2" />
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<input name="country" defaultValue="ALL" required maxLength={3} placeholder="ALL / TR / US" className="rounded-xl border border-slate-200 px-3 py-3 uppercase" />
|
||||
<select name="adActiveStatus" defaultValue="ACTIVE" className="rounded-xl border border-slate-200 bg-white px-3"><option>ACTIVE</option><option>ALL</option><option>INACTIVE</option></select>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<select name="mediaType" defaultValue="ALL" className="rounded-xl border border-slate-200 bg-white px-3"><option>ALL</option><option>VIDEO</option><option>IMAGE</option><option>MEME</option></select>
|
||||
<input name="maxResults" type="number" min="1" max="1000" defaultValue="100" required className="rounded-xl border border-slate-200 px-3 py-3" />
|
||||
</div>
|
||||
<label className="flex items-center gap-2 rounded-xl bg-slate-50 px-3 py-3 text-sm font-semibold"><input name="scrapeAdDetails" type="checkbox" defaultChecked /> Yaratıcı/CTA detaylarını al</label>
|
||||
<label className="flex items-center gap-2 rounded-xl bg-slate-50 px-3 py-3 text-sm font-semibold"><input name="includeAboutPage" type="checkbox" /> Reklamveren detaylarını al</label>
|
||||
<label className="text-sm font-semibold text-slate-600">Harcama üst sınırı (USD)<input name="maxCostUsd" type="number" min="0.1" max="10" step="0.1" defaultValue="1" required className="mt-1 w-full rounded-xl border border-slate-200 px-3 py-3" /></label>
|
||||
<button disabled={busy || !configured} className="self-end rounded-xl bg-violet-700 px-4 py-3 font-bold text-white disabled:opacity-40">{!configured ? "APIFY_TOKEN bekleniyor" : busy ? "Apify çalışıyor..." : "Reklamları çek"}</button>
|
||||
{message && <p className={`text-sm font-semibold lg:col-span-2 ${error ? "text-rose-600" : "text-emerald-600"}`}>{message}</p>}
|
||||
</form>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import { AdSource, AdStatus, MediaType, Prisma } from "@prisma/client";
|
||||
import { prisma } from "@/lib/db";
|
||||
|
||||
const APIFY_API_BASE = "https://api.apify.com/v2";
|
||||
const DEFAULT_ACTOR_ID = "solidcode~meta-ads-library-scraper";
|
||||
|
||||
export type ApifyIngestInput = {
|
||||
searchTerms: string[];
|
||||
country: string;
|
||||
adActiveStatus: "ACTIVE" | "INACTIVE" | "ALL";
|
||||
mediaType: "ALL" | "IMAGE" | "VIDEO" | "MEME" | "NONE";
|
||||
maxResults: number;
|
||||
scrapeAdDetails: boolean;
|
||||
includeAboutPage: boolean;
|
||||
maxCostUsd: number;
|
||||
};
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
function stringValue(record: JsonRecord, ...keys: string[]) {
|
||||
for (const key of keys) {
|
||||
const value = record[key];
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function numberValue(value: unknown) {
|
||||
if (typeof value === "number" && Number.isFinite(value)) return Math.round(value);
|
||||
if (typeof value === "string" && value.trim() && Number.isFinite(Number(value))) return Math.round(Number(value));
|
||||
if (value && typeof value === "object") {
|
||||
const object = value as JsonRecord;
|
||||
return numberValue(object.lower_bound ?? object.lowerBound ?? object.min);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function stringArray(record: JsonRecord, ...keys: string[]) {
|
||||
for (const key of keys) {
|
||||
const value = record[key];
|
||||
if (Array.isArray(value)) return value.filter((item): item is string => typeof item === "string" && item.length > 0);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function dateValue(value: unknown) {
|
||||
if (typeof value !== "string" || !value) return null;
|
||||
const date = new Date(value);
|
||||
return Number.isNaN(date.getTime()) ? null : date;
|
||||
}
|
||||
|
||||
function firstMediaUrl(record: JsonRecord) {
|
||||
const arrays = ["videoUrls", "videos", "imageUrls", "images", "videoPreviewImageUrls"];
|
||||
for (const key of arrays) {
|
||||
const values = stringArray(record, key);
|
||||
if (values[0]) return values[0];
|
||||
}
|
||||
return stringValue(record, "videoHdUrl", "videoUrl", "imageUrl", "adSnapshotUrl", "snapshotUrl");
|
||||
}
|
||||
|
||||
function firstThumbnailUrl(record: JsonRecord) {
|
||||
for (const key of ["videoPreviewImageUrls", "imageUrls", "images"]) {
|
||||
const values = stringArray(record, key);
|
||||
if (values[0]) return values[0];
|
||||
}
|
||||
return stringValue(record, "thumbnailUrl", "imageUrl", "pageProfilePictureURL");
|
||||
}
|
||||
|
||||
function mappedMediaType(record: JsonRecord): MediaType {
|
||||
const raw = stringValue(record, "mediaType", "media_type", "adFormat")?.toUpperCase();
|
||||
if (raw === "VIDEO") return MediaType.VIDEO;
|
||||
if (raw === "IMAGE" || raw === "MEME") return MediaType.IMAGE;
|
||||
if (raw === "CAROUSEL" || raw === "DPA") return MediaType.CAROUSEL;
|
||||
if (stringArray(record, "videoUrls", "videos").length) return MediaType.VIDEO;
|
||||
if (stringArray(record, "imageUrls", "images").length) return MediaType.IMAGE;
|
||||
return MediaType.UNKNOWN;
|
||||
}
|
||||
|
||||
export function normalizeApifyAd(record: JsonRecord) {
|
||||
const externalAdId = stringValue(record, "adArchiveID", "adArchiveId", "ad_archive_id", "id");
|
||||
if (!externalAdId) return null;
|
||||
const pageId = stringValue(record, "pageID", "pageId", "page_id");
|
||||
const pageName = stringValue(record, "pageName", "page_name", "advertiserName") || "Bilinmeyen reklamveren";
|
||||
const firstSeenAt = dateValue(record.startDate ?? record.start_date ?? record.adDeliveryStartTime);
|
||||
const endDate = dateValue(record.endDate ?? record.end_date ?? record.adDeliveryStopTime);
|
||||
const referenceDate = endDate || new Date();
|
||||
const daysRunning = firstSeenAt ? Math.max(1, Math.ceil((referenceDate.getTime() - firstSeenAt.getTime()) / 86_400_000)) : null;
|
||||
const statusText = stringValue(record, "adStatus", "status", "ad_active_status")?.toUpperCase();
|
||||
const status = statusText === "ACTIVE" ? AdStatus.ACTIVE : statusText === "INACTIVE" ? AdStatus.INACTIVE : AdStatus.UNKNOWN;
|
||||
const primaryText = stringValue(record, "adText", "primaryText", "bodyText") || stringArray(record, "adCreativeBodies", "ad_creative_bodies")[0] || null;
|
||||
const countries = stringArray(record, "countries", "reachedCountries", "ad_reached_countries");
|
||||
const country = stringValue(record, "country", "pageCountry");
|
||||
if (!countries.length && country) countries.push(country);
|
||||
|
||||
return {
|
||||
externalAdId,
|
||||
pageId,
|
||||
pageName,
|
||||
pageUrl: stringValue(record, "pageURL", "pageUrl"),
|
||||
pageLogoUrl: stringValue(record, "pageProfilePictureURL", "pageProfilePictureUrl"),
|
||||
pageLikes: numberValue(record.pageLikes),
|
||||
pageFollowers: numberValue(record.pageInstagramFollowers),
|
||||
status,
|
||||
mediaType: mappedMediaType(record),
|
||||
primaryText,
|
||||
headline: stringValue(record, "ctaHeadline", "headline", "title"),
|
||||
description: stringValue(record, "ctaDescription", "description"),
|
||||
ctaText: stringValue(record, "ctaText", "ctaType", "callToAction"),
|
||||
landingUrl: stringValue(record, "ctaUrl", "landingUrl", "linkUrl"),
|
||||
productUrl: stringValue(record, "adLibraryURL", "adLibraryUrl"),
|
||||
language: stringValue(record, "language"),
|
||||
countries,
|
||||
firstSeenAt,
|
||||
lastSeenAt: endDate,
|
||||
createdAtSource: dateValue(record.adCreationTime ?? record.ad_creation_time),
|
||||
daysRunning,
|
||||
estimatedReachMin: numberValue(record.reachEstimate),
|
||||
estimatedSpendMin: numberValue(record.spend),
|
||||
creativeUrl: firstMediaUrl(record),
|
||||
thumbnailUrl: firstThumbnailUrl(record),
|
||||
raw: record as Prisma.InputJsonValue
|
||||
};
|
||||
}
|
||||
|
||||
function apifyConfig() {
|
||||
const token = process.env.APIFY_TOKEN?.trim();
|
||||
const actorId = (process.env.APIFY_ACTOR_ID?.trim() || DEFAULT_ACTOR_ID).replace("/", "~");
|
||||
if (!token) throw new Error("APIFY_NOT_CONFIGURED");
|
||||
if (!/^[a-zA-Z0-9_-]+~[a-zA-Z0-9_-]+$/.test(actorId)) throw new Error("APIFY_ACTOR_INVALID");
|
||||
return { token, actorId };
|
||||
}
|
||||
|
||||
export async function runApifyActor(input: ApifyIngestInput) {
|
||||
const { token, actorId } = apifyConfig();
|
||||
const query = new URLSearchParams({
|
||||
clean: "true",
|
||||
timeout: "180",
|
||||
maxItems: String(input.maxResults),
|
||||
maxTotalChargeUsd: String(input.maxCostUsd)
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 210_000);
|
||||
try {
|
||||
const response = await fetch(`${APIFY_API_BASE}/actors/${actorId}/run-sync-get-dataset-items?${query}`, {
|
||||
method: "POST",
|
||||
headers: { authorization: `Bearer ${token}`, "content-type": "application/json", accept: "application/json" },
|
||||
body: JSON.stringify({
|
||||
searchTerms: input.searchTerms,
|
||||
country: input.country,
|
||||
adActiveStatus: input.adActiveStatus,
|
||||
mediaType: input.mediaType,
|
||||
adType: "ALL",
|
||||
maxResults: input.maxResults,
|
||||
scrapeAdDetails: input.scrapeAdDetails,
|
||||
includeAboutPage: input.includeAboutPage
|
||||
}),
|
||||
signal: controller.signal,
|
||||
cache: "no-store"
|
||||
});
|
||||
if (!response.ok) throw new Error(`APIFY_HTTP_${response.status}`);
|
||||
const payload: unknown = await response.json();
|
||||
if (!Array.isArray(payload)) throw new Error("APIFY_INVALID_RESPONSE");
|
||||
return payload.filter((item): item is JsonRecord => Boolean(item) && typeof item === "object" && !Array.isArray(item));
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw new Error("APIFY_TIMEOUT");
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
export async function importApifyAds(records: JsonRecord[]) {
|
||||
let imported = 0;
|
||||
let failed = 0;
|
||||
for (const record of records) {
|
||||
const item = normalizeApifyAd(record);
|
||||
if (!item) { failed += 1; continue; }
|
||||
try {
|
||||
let brandPageId: string | null = null;
|
||||
if (item.pageId) {
|
||||
const brand = await prisma.brandPage.upsert({
|
||||
where: { source_externalPageId: { source: AdSource.META, externalPageId: item.pageId } },
|
||||
create: { source: AdSource.META, externalPageId: item.pageId, name: item.pageName, pageUrl: item.pageUrl, logoUrl: item.pageLogoUrl, fbLikes: item.pageLikes, igFollowers: item.pageFollowers },
|
||||
update: { name: item.pageName, pageUrl: item.pageUrl, logoUrl: item.pageLogoUrl, fbLikes: item.pageLikes, igFollowers: item.pageFollowers }
|
||||
});
|
||||
brandPageId = brand.id;
|
||||
}
|
||||
const creative = item.creativeUrl ? { type: item.mediaType, url: item.creativeUrl, thumbnailUrl: item.thumbnailUrl } : null;
|
||||
await prisma.ad.upsert({
|
||||
where: { source_externalAdId: { source: AdSource.META, externalAdId: item.externalAdId } },
|
||||
create: {
|
||||
source: AdSource.META, externalAdId: item.externalAdId, brandPageId, status: item.status, mediaType: item.mediaType,
|
||||
primaryText: item.primaryText, headline: item.headline, description: item.description, ctaText: item.ctaText,
|
||||
landingUrl: item.landingUrl, productUrl: item.productUrl, language: item.language, countries: item.countries,
|
||||
firstSeenAt: item.firstSeenAt, lastSeenAt: item.lastSeenAt, createdAtSource: item.createdAtSource, daysRunning: item.daysRunning,
|
||||
estimatedReachMin: item.estimatedReachMin, estimatedSpendMin: item.estimatedSpendMin, raw: item.raw,
|
||||
...(creative ? { creatives: { create: creative } } : {})
|
||||
},
|
||||
update: {
|
||||
brandPageId, status: item.status, mediaType: item.mediaType, primaryText: item.primaryText, headline: item.headline,
|
||||
description: item.description, ctaText: item.ctaText, landingUrl: item.landingUrl, productUrl: item.productUrl,
|
||||
language: item.language, countries: item.countries, firstSeenAt: item.firstSeenAt, lastSeenAt: item.lastSeenAt,
|
||||
createdAtSource: item.createdAtSource, daysRunning: item.daysRunning, estimatedReachMin: item.estimatedReachMin,
|
||||
estimatedSpendMin: item.estimatedSpendMin, raw: item.raw,
|
||||
...(creative ? { creatives: { deleteMany: {}, create: creative } } : {})
|
||||
}
|
||||
});
|
||||
imported += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
return { imported, failed };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user