Node.js web scraping API for JavaScript web scraping, crawling and Puppeteer alternatives

const BASE = "https://api.clawengine.ai/v1";
const headers = {
  Authorization: `Bearer ${process.env.CLAWENGINE_API_KEY}`,
  "Content-Type": "application/json",
};

const res = await fetch(`${BASE}/extract`, {
  method: "POST",
  headers,
  body: JSON.stringify({
    url: "https://example.com/products/widget-pro",
    format: "json",
    render: true,                 // content is built client-side
    schema: {
      product_name: "string",
      price: "number",
      currency: "string",
      in_stock: "boolean",
    },
  }),
});

if (!res.ok) throw new Error(`extract failed: ${res.status}`);

const { data } = await res.json();
console.log(data.product_name, data.price, data.currency);

import { mkdir, writeFile } from "node:fs/promises";

const BASE = "https://api.clawengine.ai/v1";

const job = await fetch(`${BASE}/crawl`, {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.CLAWENGINE_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    url: "https://example.com/docs",
    include: ["/docs/*"],         // stay inside the docs tree
    max_pages: 400,
    format: "markdown",
  }),
}).then((r) => r.json());

await mkdir("./corpus", { recursive: true });

for (const page of job.pages) {
  const name = new URL(page.url).pathname.replaceAll("/", "_") + ".md";
  // markdown arrives without nav, banners or script tags, so chunk it directly
  await writeFile(`./corpus/${name}`, `# source: ${page.url}\n\n${page.markdown}`);
}

console.log(`wrote ${job.pages.length} pages`);

const BASE = "https://api.clawengine.ai/v1";
const headers = {
  Authorization: `Bearer ${process.env.CLAWENGINE_API_KEY}`,
  "Content-Type": "application/json",
};

async function extract(url) {
  const res = await fetch(`${BASE}/extract`, {
    method: "POST",
    headers,
    body: JSON.stringify({ url, format: "json", render: true,
      schema: { title: "string", published_at: "string", author: "string" } }),
    signal: AbortSignal.timeout(90_000),
  });
  if (!res.ok) return { url, error: res.status };
  return { url, ...(await res.json()).data };
}

// hand-rolled pool: keep your own concurrency sane, no p-limit needed
async function pool(urls, size = 8) {
  const queue = [...urls];
  const out = [];
  const worker = async () => {
    for (let next = queue.shift(); next; next = queue.shift()) {
      out.push(await extract(next));
    }
  };
  await Promise.all(Array.from({ length: size }, worker));
  return out;
}

const rows = await pool(["https://example.com/a", "https://example.com/b"]);
console.table(rows);