Web scraping MCP server: wire a web scraper MCP tool into Claude, Cursor and Codex

import os
import httpx2
from mcp.server import MCPServer

mcp = MCPServer("clawengine")

BASE = "https://api.clawengine.ai/v1"
HEADERS = {"Authorization": f"Bearer {os.environ['CLAWENGINE_API_KEY']}"}
MAX_PAGES = 25  # the ceiling the model cannot argue with


@mcp.tool()
async def extract_page(url: str) -> str:
    """Fetch one public web page and return it as clean markdown.

    Args:
        url: The full https URL of a single page.
    """
    async with httpx2.AsyncClient(timeout=60) as client:
        r = await client.post(
            f"{BASE}/extract",
            headers=HEADERS,
            json={"url": url, "format": "markdown"},
        )
        r.raise_for_status()
        return r.json()["markdown"]


@mcp.tool()
async def crawl_site(url: str, limit: int = 10) -> str:
    """Crawl a site section and return each page as markdown.

    Args:
        url: The starting page.
        limit: How many pages to fetch. Clamped to 25.
    """
    async with httpx2.AsyncClient(timeout=300) as client:
        r = await client.post(
            f"{BASE}/crawl",
            headers=HEADERS,
            json={"url": url, "limit": min(limit, MAX_PAGES), "format": "markdown"},
        )
        r.raise_for_status()
        return r.json()["id"]


if __name__ == "__main__":
    mcp.run(transport="stdio")

{
  "mcpServers": {
    "clawengine": {
      "command": "uv",
      "args": [
        "--directory",
        "/ABSOLUTE/PATH/TO/clawengine-mcp",
        "run",
        "clawengine_mcp.py"
      ],
      "env": {
        "CLAWENGINE_API_KEY": "your-key"
      }
    }
  }
}

# Big jobs do not belong in a tool call. Start the crawl from your own
# code, poll it to completion, and hand the agent a finished corpus.
import os, time, requests

headers = {"Authorization": f"Bearer {os.environ['CLAWENGINE_API_KEY']}"}

job = requests.post("https://api.clawengine.ai/v1/crawl", headers=headers, json={
    "url": "https://example.com/docs",
    "limit": 500,
    "format": "markdown",
}).json()

while True:
    status = requests.get(
        f"https://api.clawengine.ai/v1/crawl/{job['id']}", headers=headers
    ).json()
    if status["status"] == "completed":
        break
    time.sleep(5)

pages = status["pages"]
print(f"{len(pages)} pages, one metered fetch each, zero tool calls burned")