Web crawler API and website crawler API that renders JavaScript and turns whole sites into clean data

curl https://api.clawengine.ai/v1/crawl \
  -H "Authorization: Bearer $CLAWENGINE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com/docs",
    "include_paths": ["/docs"],
    "limit": 500,
    "max_depth": 4,
    "format": "markdown"
  }'

# 202 Accepted
# { "id": "crawl_8f2a1c9e", "status": "queued", "url": "https://example.com/docs" }

import os, time, requests

BASE = "https://api.clawengine.ai/v1"
headers = {"Authorization": f"Bearer {os.environ['CLAWENGINE_API_KEY']}"}

# 1. Kick off the crawl
job = requests.post(f"{BASE}/crawl", headers=headers, json={
    "url": "https://example.com/docs",
    "include_paths": ["/docs"],
    "limit": 500,
    "format": "markdown",
}).json()

# 2. Poll until it finishes
while True:
    res = requests.get(f"{BASE}/crawl/{job['id']}", headers=headers).json()
    if res["status"] == "completed":
        break
    time.sleep(3)

# 3. Every page is clean markdown, ready to chunk and embed
for page in res["pages"]:
    print(page["url"], len(page["markdown"]))

// Register a webhook and ClawEngine POSTs each page as it completes,
// so you never poll. Great for large or scheduled crawls.
await fetch("https://api.clawengine.ai/v1/crawl", {
  method: "POST",
  headers: {
    Authorization: `Bearer ${process.env.CLAWENGINE_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    url: "https://example.com/docs",
    limit: 500,
    format: "markdown",
    webhook: "https://yourapp.com/hooks/clawengine",
  }),
});

// Your endpoint receives { crawl_id, url, markdown, data, metadata } per page.