JavaScript Rendering API to Render and Crawl JavaScript Websites

curl https://api.clawengine.ai/v1/extract \
  -H "Authorization: Bearer $CLAWENGINE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://app.example.com/products",
    "render": true,
    "format": "markdown"
  }'

import os, requests

API = "https://api.clawengine.ai/v1/extract"
HEADERS = {"Authorization": f"Bearer {os.environ['CLAWENGINE_API_KEY']}"}

def get_page(url, min_chars=400):
    """Try the cheap path first. A near-empty body means the page
    was built client-side, so retry once with rendering on."""
    plain = requests.post(
        API, headers=HEADERS,
        json={"url": url, "render": False, "format": "markdown"},
        timeout=60,
    ).json()

    if len(plain.get("markdown", "").strip()) >= min_chars:
        return plain, False

    rendered = requests.post(
        API, headers=HEADERS,
        json={"url": url, "render": True, "format": "markdown"},
        timeout=120,
    ).json()
    return rendered, True

page, did_render = get_page("https://app.example.com/products")
print(f"rendered={did_render} chars={len(page['markdown'])}")

import os, requests

# On a client-rendered site the navigation itself is built by JavaScript,
# so the crawler has to render each page BEFORE it can discover links.
job = requests.post(
    "https://api.clawengine.ai/v1/crawl",
    headers={"Authorization": f"Bearer {os.environ['CLAWENGINE_API_KEY']}"},
    json={
        "url": "https://app.example.com",
        "render": True,          # render before link discovery
        "max_pages": 500,
        "schema": {
            "title": "string",
            "price": "number|null",
            "in_stock": "boolean",
        },
    },
    timeout=120,
).json()

print(job["id"], job["status"])