LangChain web scraping API and LangChain web scraper for scraping any website into clean documents

import os, requests
from typing import Iterator
from langchain_core.documents import Document
from langchain_core.document_loaders import BaseLoader

class ClawEngineLoader(BaseLoader):
    """Crawl a site with ClawEngine and yield LangChain Documents."""

    def __init__(self, seed_url: str, max_pages: int = 50):
        self.seed_url = seed_url
        self.max_pages = max_pages

    def lazy_load(self) -> Iterator[Document]:
        res = requests.post(
            "https://api.clawengine.ai/v1/crawl",
            headers={"Authorization": f"Bearer {os.environ['CLAWENGINE_API_KEY']}"},
            json={"url": self.seed_url, "format": "markdown",
                  "render": True, "max_pages": self.max_pages},
            timeout=300,
        )
        res.raise_for_status()
        for page in res.json()["pages"]:
            yield Document(
                page_content=page["markdown"],
                metadata={"source": page["url"], "title": page.get("title"),
                          "crawled_at": page.get("fetched_at")},
            )

# Drops into any existing chain unchanged.
docs = ClawEngineLoader("https://docs.example.com").load()

import os, requests
from langchain_core.tools import tool

@tool
def read_web_page(url: str) -> str:
    """Fetch a public web page and return it as clean markdown.

    Use this whenever you need the current contents of a URL.
    """
    res = requests.post(
        "https://api.clawengine.ai/v1/extract",
        headers={"Authorization": f"Bearer {os.environ['CLAWENGINE_API_KEY']}"},
        json={"url": url, "format": "markdown", "render": True},
        timeout=60,
    )
    res.raise_for_status()
    return res.json()["markdown"]

model_with_tools = model.bind_tools([read_web_page])