langchain-thecrawler 0.1.0__tar.gz

langchain_thecrawler-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 manchittlab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain_thecrawler-0.1.0/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.4
+Name: langchain-thecrawler
+Version: 0.1.0
+Summary: An integration package connecting TheCrawler and LangChain
+License: MIT
+License-File: LICENSE
+Author: manchittlab
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: langchain-core (>=0.3.15,<0.4.0)
+Requires-Dist: requests (>=2.31,<3.0)
+Project-URL: Homepage, https://www.miaibot.ai
+Project-URL: Repository, https://github.com/manchittlab/langchain-thecrawler
+Project-URL: Source Code, https://github.com/manchittlab/langchain-thecrawler
+Description-Content-Type: text/markdown
+
+# langchain-thecrawler
+
+This package contains the LangChain integration with [TheCrawler](https://www.miaibot.ai) — a web-scraping and structured-extraction API that runs the extraction LLM on its own GPU, so AI extraction is included on every page with no per-call surcharge.
+
+## Installation
+
+```bash
+pip install -U langchain-thecrawler
+```
+
+Set your API key (get one at [miaibot.ai](https://www.miaibot.ai)):
+
+```bash
+export THECRAWLER_API_KEY="mai_live_..."
+```
+
+## Document Loader
+
+`TheCrawlerLoader` loads one or more URLs as LangChain `Document` objects with boilerplate-stripped markdown as `page_content` and rich page metadata.
+
+```python
+from langchain_thecrawler import TheCrawlerLoader
+
+loader = TheCrawlerLoader(
+    ["https://example.com"],
+    # api_key="mai_live_...",  # or set THECRAWLER_API_KEY
+)
+
+docs = loader.load()        # list[Document]
+# or stream:
+for doc in loader.lazy_load():
+    print(doc.metadata["url"], len(doc.page_content))
+```
+
+PDF and DOCX URLs are handled server-side. A per-page failure does **not** raise — failed pages come back as a `Document` with empty `page_content` and `metadata["status"] == "error"` plus a structured `error_type`, so you can branch on it:
+
+```python
+ok = [d for d in docs if d.metadata.get("status") != "error"]
+```
+
+### Options
+
+| Arg | Description |
+| --- | --- |
+| `urls` | A URL string or list of URLs (required) |
+| `api_key` | TheCrawler key; falls back to `THECRAWLER_API_KEY` |
+| `api_url` | API base URL (default `https://www.miaibot.ai/api/v1`) |
+| `params` | Extra options merged into the crawl request (e.g. `{"usePlaywright": True}`) |
+| `timeout` | Per-request HTTP timeout in seconds (default 120) |
+
+## License
+
+MIT
+

langchain_thecrawler-0.1.0/README.md

@@ -0,0 +1,53 @@
+# langchain-thecrawler
+
+This package contains the LangChain integration with [TheCrawler](https://www.miaibot.ai) — a web-scraping and structured-extraction API that runs the extraction LLM on its own GPU, so AI extraction is included on every page with no per-call surcharge.
+
+## Installation
+
+```bash
+pip install -U langchain-thecrawler
+```
+
+Set your API key (get one at [miaibot.ai](https://www.miaibot.ai)):
+
+```bash
+export THECRAWLER_API_KEY="mai_live_..."
+```
+
+## Document Loader
+
+`TheCrawlerLoader` loads one or more URLs as LangChain `Document` objects with boilerplate-stripped markdown as `page_content` and rich page metadata.
+
+```python
+from langchain_thecrawler import TheCrawlerLoader
+
+loader = TheCrawlerLoader(
+    ["https://example.com"],
+    # api_key="mai_live_...",  # or set THECRAWLER_API_KEY
+)
+
+docs = loader.load()        # list[Document]
+# or stream:
+for doc in loader.lazy_load():
+    print(doc.metadata["url"], len(doc.page_content))
+```
+
+PDF and DOCX URLs are handled server-side. A per-page failure does **not** raise — failed pages come back as a `Document` with empty `page_content` and `metadata["status"] == "error"` plus a structured `error_type`, so you can branch on it:
+
+```python
+ok = [d for d in docs if d.metadata.get("status") != "error"]
+```
+
+### Options
+
+| Arg | Description |
+| --- | --- |
+| `urls` | A URL string or list of URLs (required) |
+| `api_key` | TheCrawler key; falls back to `THECRAWLER_API_KEY` |
+| `api_url` | API base URL (default `https://www.miaibot.ai/api/v1`) |
+| `params` | Extra options merged into the crawl request (e.g. `{"usePlaywright": True}`) |
+| `timeout` | Per-request HTTP timeout in seconds (default 120) |
+
+## License
+
+MIT

langchain_thecrawler-0.1.0/langchain_thecrawler/__init__.py

@@ -0,0 +1,7 @@
+"""LangChain integration for TheCrawler."""
+
+from langchain_thecrawler.document_loaders import TheCrawlerLoader
+
+__all__ = ["TheCrawlerLoader"]
+
+__version__ = "0.1.0"

langchain_thecrawler-0.1.0/langchain_thecrawler/document_loaders.py

@@ -0,0 +1,128 @@
+"""TheCrawler document loader for LangChain."""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Dict, Iterator, List, Optional, Union
+
+import requests
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+_DEFAULT_API_URL = "https://www.miaibot.ai/api/v1"
+
+
+class TheCrawlerLoader(BaseLoader):
+    """Load web pages as LangChain ``Document`` objects via the TheCrawler API.
+
+    TheCrawler returns boilerplate-stripped markdown plus rich page metadata
+    (title, description, status code, structured error type, ...) for each URL.
+    PDF and DOCX URLs are handled server-side. A per-page failure does **not**
+    raise: failed pages are yielded as ``Document`` objects with empty
+    ``page_content`` and ``metadata["status"] == "error"`` (plus a structured
+    ``error_type``), so caller code can branch on it instead of catching.
+
+    Args:
+        urls: A single absolute URL or a list of URLs to load.
+        api_key: TheCrawler API key (``mai_live_...``). Falls back to the
+            ``THECRAWLER_API_KEY`` environment variable.
+        api_url: Base URL of the API. Defaults to
+            ``https://www.miaibot.ai/api/v1``. Override to point at a
+            self-hosted instance.
+        params: Extra options merged into the ``/crawl`` request body (e.g.
+            ``{"usePlaywright": True, "requestTimeoutSecs": 60}``). See the
+            TheCrawler docs for the full option list.
+        timeout: Per-request HTTP timeout in seconds. Default 120.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_thecrawler import TheCrawlerLoader
+
+            loader = TheCrawlerLoader(
+                ["https://example.com"], api_key="mai_live_..."
+            )
+            docs = loader.load()
+    """
+
+    def __init__(
+        self,
+        urls: Union[str, List[str]],
+        *,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = None,
+        params: Optional[Dict[str, Any]] = None,
+        timeout: int = 120,
+    ) -> None:
+        if isinstance(urls, str):
+            urls = [urls]
+        if not urls:
+            raise ValueError("`urls` must be a non-empty URL or list of URLs.")
+
+        resolved_key = api_key or os.getenv("THECRAWLER_API_KEY", "")
+        if not resolved_key:
+            raise ValueError(
+                "TheCrawler API key is required. Pass api_key=... or set the "
+                "THECRAWLER_API_KEY environment variable."
+            )
+
+        self.urls: List[str] = list(urls)
+        self.api_key: str = resolved_key
+        self.api_url: str = (api_url or _DEFAULT_API_URL).rstrip("/")
+        self.params: Dict[str, Any] = params or {}
+        self.timeout: int = timeout
+
+    def _post_crawl(self, body: Dict[str, Any]) -> Dict[str, Any]:
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        response = requests.post(
+            f"{self.api_url}/crawl",
+            headers=headers,
+            json=body,
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    @staticmethod
+    def _page_to_document(page: Dict[str, Any]) -> Document:
+        text = page.get("markdown") or page.get("text") or ""
+        metadata: Dict[str, Any] = {
+            "source": "thecrawler",
+            "url": page.get("url", ""),
+            "title": page.get("title"),
+            "description": page.get("description"),
+            "language": page.get("language"),
+            "canonical_url": page.get("canonicalUrl"),
+            "status": page.get("status"),
+            "status_code": page.get("statusCode"),
+            "content_type": page.get("contentType"),
+            "response_time_ms": page.get("responseTimeMs"),
+            "scraped_at": page.get("scrapedAt"),
+            "from_cache": page.get("fromCache"),
+        }
+        if page.get("status") == "error":
+            metadata["error"] = page.get("error")
+            metadata["error_type"] = page.get("errorType")
+            metadata["error_retryable"] = page.get("errorRetryable")
+
+        metadata = {k: v for k, v in metadata.items() if v is not None}
+        return Document(page_content=text, metadata=metadata)
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Crawl the configured URLs and yield one ``Document`` per page."""
+        body: Dict[str, Any] = {
+            "urls": list(self.urls),
+            "extractMarkdown": True,
+            "stripBoilerplate": True,
+        }
+        # User-supplied params override defaults.
+        if self.params:
+            body.update(self.params)
+
+        result = self._post_crawl(body)
+        pages = result.get("pages", []) if isinstance(result, dict) else []
+        for page in pages:
+            yield self._page_to_document(page)

langchain_thecrawler-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[tool.poetry]
+name = "langchain-thecrawler"
+version = "0.1.0"
+description = "An integration package connecting TheCrawler and LangChain"
+authors = ["manchittlab"]
+readme = "README.md"
+license = "MIT"
+repository = "https://github.com/manchittlab/langchain-thecrawler"
+packages = [{ include = "langchain_thecrawler" }]
+
+[tool.poetry.urls]
+"Source Code" = "https://github.com/manchittlab/langchain-thecrawler"
+"Homepage" = "https://www.miaibot.ai"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<4.0"
+langchain-core = "^0.3.15"
+requests = "^2.31"
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.0"
+pytest-asyncio = "^0.23"
+pytest-socket = "^0.7"
+pytest-mock = "^3.12"
+langchain-tests = "^0.3.5"
+
+[tool.poetry.group.lint.dependencies]
+ruff = "^0.5"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"