langchain-mesharc 0.1.0__tar.gz

langchain_mesharc-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 MeshArc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain_mesharc-0.1.0/PKG-INFO

@@ -0,0 +1,127 @@
+Metadata-Version: 2.4
+Name: langchain-mesharc
+Version: 0.1.0
+Summary: LangChain integration for MeshArc: web pages as Documents, and web search and page reading as agent tools.
+License-Expression: MIT
+Project-URL: Homepage, https://mesharc.dev
+Project-URL: Documentation, https://github.com/mesharc-org/langchain-mesharc#readme
+Project-URL: Source, https://github.com/mesharc-org/langchain-mesharc
+Project-URL: Issues, https://github.com/mesharc-org/langchain-mesharc/issues
+Project-URL: Privacy, https://mesharc.dev/legal/privacy
+Keywords: langchain,mesharc,web-scraping,crawler,document-loader,rag,markdown,change-detection
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Internet :: WWW/HTTP
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: langchain-core<2,>=1.0
+Requires-Dist: mesharc<1,>=0.6
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: langchain-tests==1.1.9; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: ruff>=0.16; extra == "dev"
+Requires-Dist: mypy>=1.18; extra == "dev"
+Dynamic: license-file
+
+# langchain-mesharc
+
+The LangChain integration for [MeshArc](https://mesharc.dev), the web crawler that turns sites into clean markdown and records what changed between crawls.
+
+- **`MeshArcLoader`** loads web pages as LangChain `Document`s, in four modes: scrape, crawl, map and changes.
+- **`MeshArcSearch`** and **`MeshArcScrape`** are tools an agent can call to search the web and read pages.
+
+```bash
+pip install langchain-mesharc
+```
+
+You need a MeshArc API key. Make one under **Settings → API keys** at [mesharc.dev](https://mesharc.dev); the free plan includes 1,000 credits with no card. Set it as `MESHARC_API_KEY`, or pass `api_key=`. Reading pages and searching need a key with **write** access.
+
+## Document loader
+
+```python
+from langchain_mesharc import MeshArcLoader
+
+# One page, or many read as one batch
+docs = MeshArcLoader("https://example.com/pricing").load()
+docs = MeshArcLoader(["https://example.com/", "https://example.com/about"]).load()
+
+# A whole site, or a section of it; pages arrive as they are read
+loader = MeshArcLoader("https://docs.example.com", mode="crawl",
+                       params={"limit": 100, "includePaths": ["/guides/*"]})
+for doc in loader.lazy_load():
+    print(doc.metadata["url"], len(doc.page_content))
+
+# The URLs a site declares in its sitemaps, without fetching them
+urls = MeshArcLoader("https://docs.example.com", mode="map").load()
+
+# What changed on a watched site since the last crawl
+changed = MeshArcLoader(mode="changes", project_id="prj_...").load()
+```
+
+| Mode | Input | Each Document |
+|---|---|---|
+| `scrape` | a URL or a list of URLs | the page's markdown |
+| `crawl` | a site's URL; `params` takes `limit`, `maxDepth`, `includePaths`, `excludePaths` and the rest of the [crawl options](https://mesharc.dev/docs/api) | one per page read |
+| `map` | a site's URL; `params` takes `search` and `limit` | one per URL in the sitemaps, with the URL as its text |
+| `changes` | `project_id`, and optionally `run_id` and `against` | one per added or modified page, holding the page as it is now |
+
+**Metadata** on every page: `source` and `url`, `title`, `page_status` (one word: `ok`, `missing`, `blocked`, `captcha`, …), `crawled_at` and `mode`.
+- Scraped and changed pages also have `status`, the HTTP status. A crawl's pages don't, because the API leaves it out of crawl results.
+- Scraped pages also have `verdict` and `credits`.
+- A crawl adds `crawl_id`.
+- Changes add `project_id`, `run_id`, `previous_run_id`, `changed_at`, `change` (`added`, `modified`, `removed` or `fields`), `words_before` and `words_after`.
+
+Values are always strings, numbers or booleans, so any vector store accepts them.
+
+Pages that came back with no content (blocked, or a 404) are left out. Pass `include_empty=True` to keep them; their `page_status` and `error_code` say why. In `changes` mode, removed pages are left out unless `include_removed=True`.
+
+`changes` mode keeps a vector store current: load it after each run of a [MeshArc project](https://mesharc.dev/docs), and re-embed only the pages that changed. Each Document's `id` is `run_id:url`.
+
+## Tools
+
+```python
+from langchain_mesharc import MeshArcScrape, MeshArcSearch
+
+tools = [MeshArcSearch(), MeshArcScrape()]
+```
+
+| Tool | The agent gives | It gets back |
+|---|---|---|
+| `mesharc_search` (`MeshArcSearch`) | `query`, `limit` (1–10), `read_pages` | a numbered list of results (title, URL, snippet), and with `read_pages` each result's page text |
+| `mesharc_scrape` (`MeshArcScrape`) | `urls` (1–20) | each page's title, URL and markdown, or why a page could not be read |
+
+Both tools return `content_and_artifact`:
+- the **content** is the text sent to the model, with each page cut at `max_chars_per_page` (8,000 by default; 0 for no limit);
+- the **artifact** is MeshArc's full response.
+
+A refused request, such as one with too few credits left, comes back to the agent as an error message rather than stopping it.
+
+## Cost
+
+Requests are billed in MeshArc credits, the same as any other API call:
+- a plain page read costs 1 credit, and blocked pages and 404s cost nothing;
+- a web search costs what reading its results page costs;
+- a sitemap read costs 1 credit per sitemap file.
+
+See [pricing](https://mesharc.dev/pricing).
+
+## Links
+
+- [MeshArc docs](https://mesharc.dev/docs) and [API reference](https://mesharc.dev/docs/api)
+- [The Python SDK](https://github.com/mesharc-org/mesharc-python), which this package uses
+- [Questions](https://github.com/orgs/mesharc-org/discussions) · security reports to security@mesharc.dev
+
+## License
+
+MIT

langchain_mesharc-0.1.0/README.md

@@ -0,0 +1,91 @@
+# langchain-mesharc
+
+The LangChain integration for [MeshArc](https://mesharc.dev), the web crawler that turns sites into clean markdown and records what changed between crawls.
+
+- **`MeshArcLoader`** loads web pages as LangChain `Document`s, in four modes: scrape, crawl, map and changes.
+- **`MeshArcSearch`** and **`MeshArcScrape`** are tools an agent can call to search the web and read pages.
+
+```bash
+pip install langchain-mesharc
+```
+
+You need a MeshArc API key. Make one under **Settings → API keys** at [mesharc.dev](https://mesharc.dev); the free plan includes 1,000 credits with no card. Set it as `MESHARC_API_KEY`, or pass `api_key=`. Reading pages and searching need a key with **write** access.
+
+## Document loader
+
+```python
+from langchain_mesharc import MeshArcLoader
+
+# One page, or many read as one batch
+docs = MeshArcLoader("https://example.com/pricing").load()
+docs = MeshArcLoader(["https://example.com/", "https://example.com/about"]).load()
+
+# A whole site, or a section of it; pages arrive as they are read
+loader = MeshArcLoader("https://docs.example.com", mode="crawl",
+                       params={"limit": 100, "includePaths": ["/guides/*"]})
+for doc in loader.lazy_load():
+    print(doc.metadata["url"], len(doc.page_content))
+
+# The URLs a site declares in its sitemaps, without fetching them
+urls = MeshArcLoader("https://docs.example.com", mode="map").load()
+
+# What changed on a watched site since the last crawl
+changed = MeshArcLoader(mode="changes", project_id="prj_...").load()
+```
+
+| Mode | Input | Each Document |
+|---|---|---|
+| `scrape` | a URL or a list of URLs | the page's markdown |
+| `crawl` | a site's URL; `params` takes `limit`, `maxDepth`, `includePaths`, `excludePaths` and the rest of the [crawl options](https://mesharc.dev/docs/api) | one per page read |
+| `map` | a site's URL; `params` takes `search` and `limit` | one per URL in the sitemaps, with the URL as its text |
+| `changes` | `project_id`, and optionally `run_id` and `against` | one per added or modified page, holding the page as it is now |
+
+**Metadata** on every page: `source` and `url`, `title`, `page_status` (one word: `ok`, `missing`, `blocked`, `captcha`, …), `crawled_at` and `mode`.
+- Scraped and changed pages also have `status`, the HTTP status. A crawl's pages don't, because the API leaves it out of crawl results.
+- Scraped pages also have `verdict` and `credits`.
+- A crawl adds `crawl_id`.
+- Changes add `project_id`, `run_id`, `previous_run_id`, `changed_at`, `change` (`added`, `modified`, `removed` or `fields`), `words_before` and `words_after`.
+
+Values are always strings, numbers or booleans, so any vector store accepts them.
+
+Pages that came back with no content (blocked, or a 404) are left out. Pass `include_empty=True` to keep them; their `page_status` and `error_code` say why. In `changes` mode, removed pages are left out unless `include_removed=True`.
+
+`changes` mode keeps a vector store current: load it after each run of a [MeshArc project](https://mesharc.dev/docs), and re-embed only the pages that changed. Each Document's `id` is `run_id:url`.
+
+## Tools
+
+```python
+from langchain_mesharc import MeshArcScrape, MeshArcSearch
+
+tools = [MeshArcSearch(), MeshArcScrape()]
+```
+
+| Tool | The agent gives | It gets back |
+|---|---|---|
+| `mesharc_search` (`MeshArcSearch`) | `query`, `limit` (1–10), `read_pages` | a numbered list of results (title, URL, snippet), and with `read_pages` each result's page text |
+| `mesharc_scrape` (`MeshArcScrape`) | `urls` (1–20) | each page's title, URL and markdown, or why a page could not be read |
+
+Both tools return `content_and_artifact`:
+- the **content** is the text sent to the model, with each page cut at `max_chars_per_page` (8,000 by default; 0 for no limit);
+- the **artifact** is MeshArc's full response.
+
+A refused request, such as one with too few credits left, comes back to the agent as an error message rather than stopping it.
+
+## Cost
+
+Requests are billed in MeshArc credits, the same as any other API call:
+- a plain page read costs 1 credit, and blocked pages and 404s cost nothing;
+- a web search costs what reading its results page costs;
+- a sitemap read costs 1 credit per sitemap file.
+
+See [pricing](https://mesharc.dev/pricing).
+
+## Links
+
+- [MeshArc docs](https://mesharc.dev/docs) and [API reference](https://mesharc.dev/docs/api)
+- [The Python SDK](https://github.com/mesharc-org/mesharc-python), which this package uses
+- [Questions](https://github.com/orgs/mesharc-org/discussions) · security reports to security@mesharc.dev
+
+## License
+
+MIT

langchain_mesharc-0.1.0/langchain_mesharc/__init__.py

@@ -0,0 +1,7 @@
+"""LangChain integration for MeshArc: web pages as Documents, and web search and page reading as agent tools."""
+
+from langchain_mesharc.document_loaders import MeshArcLoader
+from langchain_mesharc.tools import MeshArcScrape, MeshArcSearch
+
+__version__ = "0.1.0"
+__all__ = ["MeshArcLoader", "MeshArcScrape", "MeshArcSearch", "__version__"]

langchain_mesharc-0.1.0/langchain_mesharc/_core.py

@@ -0,0 +1,192 @@
+"""MeshArc results as plain document records, independent of any framework.
+
+langchain-mesharc and llama-index-readers-mesharc carry this same file; each
+wraps the records in its own Document class. Keep the two copies identical.
+
+A record is ``{"id": str, "text": str, "metadata": dict}``. Metadata values
+are str, int or bool only, never None, so every vector store accepts them.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, Iterator, Optional, Union
+
+MODES = ("scrape", "crawl", "map", "changes")
+CHANGE_KINDS = ("added", "modified", "removed", "fields")
+
+Record = Dict[str, Any]
+
+
+def make_client(api_key: Optional[str] = None, client: Any = None) -> Any:
+    """The client to call: the one given, or a MeshArc client for ``api_key``
+    (falling back to the ``MESHARC_API_KEY`` environment variable)."""
+    if client is not None:
+        return client
+    from mesharc import MeshArc
+
+    return MeshArc(api_key)
+
+
+def _clean(metadata: Dict[str, Any]) -> Dict[str, Any]:
+    """Drop empty values; keep str, int and bool as they are, anything else as str."""
+    out: Dict[str, Any] = {}
+    for key, value in metadata.items():
+        if value is None or value == "":
+            continue
+        out[key] = value if isinstance(value, (str, int, bool)) else str(value)
+    return out
+
+
+def page_metadata(row: Dict[str, Any], mode: str) -> Dict[str, Any]:
+    """What a page row says about itself, under the names every mode shares.
+
+    The API returns pages in two shapes. A scrape's rows carry the HTTP code
+    as ``httpStatus``, the engine's ``verdict`` and the one-word page status
+    as ``status``. A stored page (a crawl's rows, a project's page) carries
+    the HTTP code as ``status`` and the word as ``pageStatus`` -- and a
+    crawl's rows then overwrite ``status`` with the word. Both read alike here.
+    """
+    head = row.get("head") or {}
+    url = row.get("url") or ""
+    raw_status = row.get("status")
+    http = row.get("httpStatus")
+    if http is None and isinstance(raw_status, int) and not isinstance(raw_status, bool):
+        http = raw_status
+    word = row.get("pageStatus") or (raw_status if isinstance(raw_status, str) else "")
+    return {
+        "source": url,
+        "url": url,
+        "title": row.get("title") or head.get("title") or "",
+        "status": int(http or 0) or None,
+        "page_status": word,
+        "verdict": row.get("verdict") or "",
+        "error_code": row.get("errorCode") or "",
+        "words": int(row.get("words") or 0) or None,
+        "language": row.get("language") or "",
+        "crawled_at": row.get("crawledAt") or "",
+        "credits": row.get("credits"),
+        "mode": mode,
+    }
+
+
+def page_record(row: Dict[str, Any], mode: str, **extra: Any) -> Record:
+    metadata = page_metadata(row, mode)
+    metadata.update(extra)
+    url = row.get("url") or ""
+    return {"id": url, "text": row.get("markdown") or "", "metadata": _clean(metadata)}
+
+
+def batch_rows(batch: Optional[Dict[str, Any]]) -> list:
+    """The rows of a finished batch: one per URL, under ``pages`` (a batch
+    is not an envelope, so it has no ``data``)."""
+    return list((batch or {}).get("pages") or [])
+
+
+def scrape(client: Any, urls: Union[str, Iterable[str]], params: Dict[str, Any],
+           include_empty: bool = False) -> Iterator[Record]:
+    """One URL, or many as one batch. Each page row becomes a record."""
+    if isinstance(urls, str):
+        rows = [client.scrape(urls, **params)]
+    else:
+        rows = batch_rows(client.scrape(list(urls), **params))
+    for row in rows:
+        record = page_record(row, "scrape")
+        if record["text"] or include_empty:
+            yield record
+
+
+def crawl(client: Any, url: str, params: Dict[str, Any], include_empty: bool = False) -> Iterator[Record]:
+    """A one-shot crawl of a site. Pages are yielded as they land."""
+    job = client.crawl(url, **params)
+    for row in job.pages():
+        record = page_record(row, "crawl", crawl_id=job.id)
+        if record["text"] or include_empty:
+            yield record
+
+
+def site_map(client: Any, url: str, params: Dict[str, Any]) -> Iterator[Record]:
+    """The URLs a site declares in its sitemaps. The text is the URL; nothing is fetched."""
+    for row in client.map(url, **params):
+        page_url = row.get("url") or ""
+        metadata = {
+            "source": page_url,
+            "url": page_url,
+            "lastmod": row.get("lastmod") or "",
+            "changefreq": row.get("changefreq") or "",
+            "section": row.get("section") or "",
+            "sitemap": row.get("file") or "",
+            "mode": "map",
+        }
+        yield {"id": page_url, "text": page_url, "metadata": _clean(metadata)}
+
+
+def changes(client: Any, project_id: str, run_id: Optional[str] = None, against: Optional[str] = None,
+            include_removed: bool = False, include_empty: bool = False) -> Iterator[Record]:
+    """What changed in a project's run against the run before it (or against
+    ``against``). Each added, modified or fields-only page becomes a record
+    holding the page as it is now; a removed page has no text and is
+    yielded only with ``include_removed``."""
+    change = (client.changes(project_id, run_id=run_id, against=against) or {}).get("change")
+    if not change:
+        return
+    run = change.get("runId") or run_id or ""
+    shared = {
+        "project_id": project_id,
+        "run_id": run,
+        "previous_run_id": change.get("prevRunId") or "",
+        "changed_at": change.get("createdAt") or "",
+    }
+    for entry in change.get("feed") or []:
+        kind = entry.get("kind") or ""
+        url = entry.get("url") or ""
+        about = dict(shared, change=kind, words_before=int(entry.get("wordsBefore") or 0),
+                     words_after=int(entry.get("wordsAfter") or 0))
+        if kind == "removed":
+            if include_removed:
+                metadata = dict(about, source=url, url=url, title=entry.get("section") or "", mode="changes")
+                yield {"id": f"{run}:{url}", "text": "", "metadata": _clean(metadata)}
+            continue
+        try:
+            page = client.page(project_id, url, run_id=run or None)
+        except Exception as exc:
+            # The stored copy is gone (pruned by the project's retention, or
+            # the run deleted): skip the page rather than end the whole load.
+            if getattr(exc, "status", None) != 404:
+                raise
+            if include_empty:
+                metadata = dict(about, source=url, url=url, title=entry.get("section") or "", mode="changes")
+                yield {"id": f"{run}:{url}", "text": "", "metadata": _clean(metadata)}
+            continue
+        record = page_record(dict(page, url=page.get("url") or url), "changes", **about)
+        record["id"] = f"{run}:{url}"
+        if record["text"] or include_empty:
+            yield record
+
+
+def load(client: Any, mode: str, url: Union[str, Iterable[str], None] = None, project_id: Optional[str] = None,
+         run_id: Optional[str] = None, against: Optional[str] = None, params: Optional[Dict[str, Any]] = None,
+         include_empty: bool = False, include_removed: bool = False) -> Iterator[Record]:
+    """Records for one of the four modes."""
+    check(mode, url, project_id)
+    extra = dict(params or {})
+    if mode == "scrape":
+        return scrape(client, url, extra, include_empty)  # type: ignore[arg-type]
+    if mode == "crawl":
+        return crawl(client, url, extra, include_empty)  # type: ignore[arg-type]
+    if mode == "map":
+        return site_map(client, url, extra)  # type: ignore[arg-type]
+    return changes(client, project_id, run_id, against, include_removed, include_empty)  # type: ignore[arg-type]
+
+
+def check(mode: str, url: Union[str, Iterable[str], None], project_id: Optional[str]) -> None:
+    """Refuse a combination that cannot work, before any request is made."""
+    if mode not in MODES:
+        raise ValueError(f"mode must be one of {', '.join(MODES)}, not {mode!r}")
+    if mode == "changes":
+        if not project_id:
+            raise ValueError("mode 'changes' needs project_id: the MeshArc project whose changes to load")
+        return
+    if not url:
+        raise ValueError(f"mode {mode!r} needs url")
+    if mode in ("crawl", "map") and not isinstance(url, str):
+        raise ValueError(f"mode {mode!r} takes one url, a string")

langchain_mesharc-0.1.0/langchain_mesharc/document_loaders.py

@@ -0,0 +1,100 @@
+"""MeshArcLoader: pages from MeshArc as LangChain Documents."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterator, Literal, Optional, Sequence, Union
+
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+from . import _core
+
+
+class MeshArcLoader(BaseLoader):
+    """Load web pages through MeshArc as Documents of clean markdown.
+
+    Modes:
+        * ``scrape``: one URL, or a list of URLs read as one batch.
+        * ``crawl``: a one-shot crawl of a site, pages yielded as they land.
+        * ``map``: the URLs the site declares in its sitemaps, one Document
+          per URL whose text is the URL itself. Nothing is fetched.
+        * ``changes``: what changed in a MeshArc project's latest run (or
+          ``run_id``) against the run before it (or ``against``). Each
+          added or modified page is a Document of the page as it is now.
+
+    Metadata: ``source``/``url``, ``title``, ``page_status`` (one word:
+    ok, missing, blocked, captcha, ...), ``crawled_at`` and ``mode``;
+    scraped and changed pages add ``status`` (the HTTP status), scraped
+    pages ``verdict`` and ``credits``, crawls ``crawl_id``, and changes
+    ``project_id``, ``run_id``, ``previous_run_id``, ``changed_at``,
+    ``change`` (added, modified, removed or fields) and
+    ``words_before``/``words_after``.
+
+    Pages that came back without content (blocked, a 404) are skipped
+    unless ``include_empty`` is set; their ``page_status`` and
+    ``error_code`` then say why. ``params`` are passed to the MeshArc SDK call as they are,
+    for example ``{"limit": 50, "includePaths": ["/blog/*"]}`` for a crawl
+    or ``{"search": "pricing"}`` for a map.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_mesharc import MeshArcLoader
+
+            docs = MeshArcLoader("https://example.com/pricing").load()
+            docs = MeshArcLoader("https://docs.example.com", mode="crawl",
+                                 params={"limit": 50}).load()
+            changed = MeshArcLoader(mode="changes", project_id="prj_...").load()
+    """
+
+    def __init__(
+        self,
+        url: Union[str, Sequence[str], None] = None,
+        *,
+        mode: Literal["scrape", "crawl", "map", "changes"] = "scrape",
+        project_id: Optional[str] = None,
+        run_id: Optional[str] = None,
+        against: Optional[str] = None,
+        params: Optional[Dict[str, Any]] = None,
+        include_empty: bool = False,
+        include_removed: bool = False,
+        api_key: Optional[str] = None,
+        client: Any = None,
+    ) -> None:
+        """
+        Args:
+            url: The URL (or, for ``scrape``, a list of URLs). Not used by ``changes``.
+            mode: ``scrape``, ``crawl``, ``map`` or ``changes``.
+            project_id: The MeshArc project, for ``changes``.
+            run_id: For ``changes``: the run to report on; the latest finished one by default.
+            against: For ``changes``: compare with this run instead of the one before.
+            params: Extra arguments for the MeshArc SDK call.
+            include_empty: Also yield pages that came back without content.
+            include_removed: For ``changes``: also yield removed pages, with no text.
+            api_key: A MeshArc API key; ``MESHARC_API_KEY`` is used when omitted.
+            client: A ``mesharc.MeshArc`` client to use instead of making one.
+        """
+        _core.check(mode, url, project_id)
+        self.url = url
+        self.mode = mode
+        self.project_id = project_id
+        self.run_id = run_id
+        self.against = against
+        self.params = dict(params or {})
+        self.include_empty = include_empty
+        self.include_removed = include_removed
+        self._client = _core.make_client(api_key, client)
+
+    def lazy_load(self) -> Iterator[Document]:
+        for record in _core.load(
+            self._client,
+            self.mode,
+            url=self.url,
+            project_id=self.project_id,
+            run_id=self.run_id,
+            against=self.against,
+            params=self.params,
+            include_empty=self.include_empty,
+            include_removed=self.include_removed,
+        ):
+            yield Document(page_content=record["text"], metadata=record["metadata"], id=record["id"])

langchain_mesharc-0.1.0/langchain_mesharc/tools.py

@@ -0,0 +1,156 @@
+"""MeshArc tools for LangChain agents: search the web, and read pages."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple, Type
+
+from langchain_core.callbacks import CallbackManagerForToolRun
+from langchain_core.tools import BaseTool, ToolException
+from pydantic import BaseModel, Field, PrivateAttr, SecretStr, model_validator
+
+from . import _core
+
+DEFAULT_MAX_CHARS = 8000
+
+
+def _error(exc: Exception) -> ToolException:
+    """A refused request, as a message the model can act on."""
+    code = getattr(exc, "code", "")
+    return ToolException(f"MeshArc refused the request{f' ({code})' if code else ''}: {exc}")
+
+
+def _clip(text: str, limit: int) -> str:
+    if limit and len(text) > limit:
+        return text[:limit].rstrip() + f"\n\n[... cut at {limit} characters]"
+    return text
+
+
+class _MeshArcTool(BaseTool):
+    """What both tools share: the API key and the client."""
+
+    api_key: Optional[SecretStr] = Field(default=None, exclude=True)
+    """A MeshArc API key; ``MESHARC_API_KEY`` is used when omitted."""
+    client: Any = Field(default=None, exclude=True)
+    """A ``mesharc.MeshArc`` client to use instead of making one."""
+    max_chars_per_page: int = DEFAULT_MAX_CHARS
+    """How much of each page's markdown goes to the model; 0 for all of it."""
+    handle_tool_error: bool = True  # type: ignore[assignment]
+    response_format: str = "content_and_artifact"  # type: ignore[assignment]
+
+    _client: Any = PrivateAttr(default=None)
+
+    @model_validator(mode="after")
+    def _connect(self) -> "_MeshArcTool":
+        key = self.api_key.get_secret_value() if self.api_key else None
+        self._client = _core.make_client(key, self.client)
+        return self
+
+
+class MeshArcSearchInput(BaseModel):
+    query: str = Field(description="What to search the web for, as you would type it into a search engine.")
+    limit: int = Field(default=5, ge=1, le=10, description="How many results to return, 1 to 10.")
+    read_pages: bool = Field(
+        default=False,
+        description="Also read each result's page and return its text. Slower and costs more credits; "
+        "use it when the snippets are not enough to answer.",
+    )
+
+
+class MeshArcSearch(_MeshArcTool):
+    """Search the web through MeshArc.
+
+    Returns the organic results (title, URL, snippet) of a web search, and
+    with ``read_pages`` the text of each result's page. The content sent to
+    the model is a numbered list; the artifact is MeshArc's full response.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_mesharc import MeshArcSearch
+
+            tool = MeshArcSearch()
+            tool.invoke({"query": "rail funding consultation 2026", "limit": 3})
+    """
+
+    name: str = "mesharc_search"
+    description: str = (
+        "Search the web. Returns the top results with title, URL and snippet; set read_pages to also get "
+        "the text of each result's page. Use for current events, facts to verify, or finding pages to read."
+    )
+    args_schema: Type[BaseModel] = MeshArcSearchInput
+
+    def _run(self, query: str, limit: int = 5, read_pages: bool = False,
+             run_manager: Optional[CallbackManagerForToolRun] = None) -> Tuple[str, Dict[str, Any]]:
+        try:
+            out = self._client.web_search(query, limit=limit, scrape=True if read_pages else None)
+        except Exception as exc:  # the SDK's MeshArcError, a timeout or a network failure
+            raise _error(exc) from exc
+        if out.get("status") == "blocked":
+            return "The search engines refused this search; nothing was charged. Try again later.", out
+        hits = out.get("data") or []
+        if not hits:
+            return f"No results for {query!r}.", out
+        blocks: List[str] = []
+        for hit in hits:
+            lines = [f"{hit.get('position', len(blocks) + 1)}. {hit.get('title') or hit.get('url')}",
+                     f"   {hit.get('url', '')}"]
+            if hit.get("snippet"):
+                lines.append(f"   {hit['snippet']}")
+            page = hit.get("page") or {}
+            if read_pages and page.get("markdown"):
+                lines.append("")
+                lines.append(_clip(page["markdown"], self.max_chars_per_page))
+            blocks.append("\n".join(lines))
+        return "\n\n".join(blocks), out
+
+
+class MeshArcScrapeInput(BaseModel):
+    urls: List[str] = Field(
+        min_length=1, max_length=20,
+        description="The full URLs (with https://) of the pages to read, 1 to 20.",
+    )
+
+
+class MeshArcScrape(_MeshArcTool):
+    """Read web pages through MeshArc as clean markdown.
+
+    Reads each URL (past bot walls when needed) and returns its text. The
+    content sent to the model is each page's title, URL and markdown, cut at
+    ``max_chars_per_page``; the artifact is the list of MeshArc page rows.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_mesharc import MeshArcScrape
+
+            tool = MeshArcScrape()
+            tool.invoke({"urls": ["https://example.com/pricing"]})
+    """
+
+    name: str = "mesharc_scrape"
+    description: str = (
+        "Read web pages and return their main text as markdown. Give the full URLs. "
+        "Use when you know which pages you need; use mesharc_search first to find them."
+    )
+    args_schema: Type[BaseModel] = MeshArcScrapeInput
+
+    def _run(self, urls: List[str],
+             run_manager: Optional[CallbackManagerForToolRun] = None) -> Tuple[str, List[Dict[str, Any]]]:
+        try:
+            if len(urls) == 1:
+                rows = [self._client.scrape(urls[0])]
+            else:
+                rows = _core.batch_rows(self._client.scrape(list(urls)))
+        except Exception as exc:
+            raise _error(exc) from exc
+        blocks: List[str] = []
+        for row in rows:
+            meta = _core.page_metadata(row, "scrape")
+            if row.get("markdown"):
+                head = f"# {meta['title']}\n" if meta["title"] else ""
+                blocks.append(f"{head}URL: {meta['url']}\n\n{_clip(row['markdown'], self.max_chars_per_page)}")
+            else:
+                why = (row.get("reason") or row.get("error") or meta["error_code"] or meta["page_status"]
+                       or meta["verdict"] or "no content")
+                blocks.append(f"URL: {meta['url']}\nNot read: {why}.")
+        return "\n\n---\n\n".join(blocks) or "Nothing came back.", rows

langchain_mesharc-0.1.0/langchain_mesharc.egg-info/PKG-INFO

@@ -0,0 +1,127 @@
+Metadata-Version: 2.4
+Name: langchain-mesharc
+Version: 0.1.0
+Summary: LangChain integration for MeshArc: web pages as Documents, and web search and page reading as agent tools.
+License-Expression: MIT
+Project-URL: Homepage, https://mesharc.dev
+Project-URL: Documentation, https://github.com/mesharc-org/langchain-mesharc#readme
+Project-URL: Source, https://github.com/mesharc-org/langchain-mesharc
+Project-URL: Issues, https://github.com/mesharc-org/langchain-mesharc/issues
+Project-URL: Privacy, https://mesharc.dev/legal/privacy
+Keywords: langchain,mesharc,web-scraping,crawler,document-loader,rag,markdown,change-detection
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Internet :: WWW/HTTP
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: langchain-core<2,>=1.0
+Requires-Dist: mesharc<1,>=0.6
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: langchain-tests==1.1.9; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: ruff>=0.16; extra == "dev"
+Requires-Dist: mypy>=1.18; extra == "dev"
+Dynamic: license-file
+
+# langchain-mesharc
+
+The LangChain integration for [MeshArc](https://mesharc.dev), the web crawler that turns sites into clean markdown and records what changed between crawls.
+
+- **`MeshArcLoader`** loads web pages as LangChain `Document`s, in four modes: scrape, crawl, map and changes.
+- **`MeshArcSearch`** and **`MeshArcScrape`** are tools an agent can call to search the web and read pages.
+
+```bash
+pip install langchain-mesharc
+```
+
+You need a MeshArc API key. Make one under **Settings → API keys** at [mesharc.dev](https://mesharc.dev); the free plan includes 1,000 credits with no card. Set it as `MESHARC_API_KEY`, or pass `api_key=`. Reading pages and searching need a key with **write** access.
+
+## Document loader
+
+```python
+from langchain_mesharc import MeshArcLoader
+
+# One page, or many read as one batch
+docs = MeshArcLoader("https://example.com/pricing").load()
+docs = MeshArcLoader(["https://example.com/", "https://example.com/about"]).load()
+
+# A whole site, or a section of it; pages arrive as they are read
+loader = MeshArcLoader("https://docs.example.com", mode="crawl",
+                       params={"limit": 100, "includePaths": ["/guides/*"]})
+for doc in loader.lazy_load():
+    print(doc.metadata["url"], len(doc.page_content))
+
+# The URLs a site declares in its sitemaps, without fetching them
+urls = MeshArcLoader("https://docs.example.com", mode="map").load()
+
+# What changed on a watched site since the last crawl
+changed = MeshArcLoader(mode="changes", project_id="prj_...").load()
+```
+
+| Mode | Input | Each Document |
+|---|---|---|
+| `scrape` | a URL or a list of URLs | the page's markdown |
+| `crawl` | a site's URL; `params` takes `limit`, `maxDepth`, `includePaths`, `excludePaths` and the rest of the [crawl options](https://mesharc.dev/docs/api) | one per page read |
+| `map` | a site's URL; `params` takes `search` and `limit` | one per URL in the sitemaps, with the URL as its text |
+| `changes` | `project_id`, and optionally `run_id` and `against` | one per added or modified page, holding the page as it is now |
+
+**Metadata** on every page: `source` and `url`, `title`, `page_status` (one word: `ok`, `missing`, `blocked`, `captcha`, …), `crawled_at` and `mode`.
+- Scraped and changed pages also have `status`, the HTTP status. A crawl's pages don't, because the API leaves it out of crawl results.
+- Scraped pages also have `verdict` and `credits`.
+- A crawl adds `crawl_id`.
+- Changes add `project_id`, `run_id`, `previous_run_id`, `changed_at`, `change` (`added`, `modified`, `removed` or `fields`), `words_before` and `words_after`.
+
+Values are always strings, numbers or booleans, so any vector store accepts them.
+
+Pages that came back with no content (blocked, or a 404) are left out. Pass `include_empty=True` to keep them; their `page_status` and `error_code` say why. In `changes` mode, removed pages are left out unless `include_removed=True`.
+
+`changes` mode keeps a vector store current: load it after each run of a [MeshArc project](https://mesharc.dev/docs), and re-embed only the pages that changed. Each Document's `id` is `run_id:url`.
+
+## Tools
+
+```python
+from langchain_mesharc import MeshArcScrape, MeshArcSearch
+
+tools = [MeshArcSearch(), MeshArcScrape()]
+```
+
+| Tool | The agent gives | It gets back |
+|---|---|---|
+| `mesharc_search` (`MeshArcSearch`) | `query`, `limit` (1–10), `read_pages` | a numbered list of results (title, URL, snippet), and with `read_pages` each result's page text |
+| `mesharc_scrape` (`MeshArcScrape`) | `urls` (1–20) | each page's title, URL and markdown, or why a page could not be read |
+
+Both tools return `content_and_artifact`:
+- the **content** is the text sent to the model, with each page cut at `max_chars_per_page` (8,000 by default; 0 for no limit);
+- the **artifact** is MeshArc's full response.
+
+A refused request, such as one with too few credits left, comes back to the agent as an error message rather than stopping it.
+
+## Cost
+
+Requests are billed in MeshArc credits, the same as any other API call:
+- a plain page read costs 1 credit, and blocked pages and 404s cost nothing;
+- a web search costs what reading its results page costs;
+- a sitemap read costs 1 credit per sitemap file.
+
+See [pricing](https://mesharc.dev/pricing).
+
+## Links
+
+- [MeshArc docs](https://mesharc.dev/docs) and [API reference](https://mesharc.dev/docs/api)
+- [The Python SDK](https://github.com/mesharc-org/mesharc-python), which this package uses
+- [Questions](https://github.com/orgs/mesharc-org/discussions) · security reports to security@mesharc.dev
+
+## License
+
+MIT

langchain_mesharc-0.1.0/langchain_mesharc.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+README.md
+pyproject.toml
+langchain_mesharc/__init__.py
+langchain_mesharc/_core.py
+langchain_mesharc/document_loaders.py
+langchain_mesharc/py.typed
+langchain_mesharc/tools.py
+langchain_mesharc.egg-info/PKG-INFO
+langchain_mesharc.egg-info/SOURCES.txt
+langchain_mesharc.egg-info/dependency_links.txt
+langchain_mesharc.egg-info/requires.txt
+langchain_mesharc.egg-info/top_level.txt

langchain_mesharc-0.1.0/langchain_mesharc.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

langchain_mesharc-0.1.0/langchain_mesharc.egg-info/requires.txt

@@ -0,0 +1,11 @@
+langchain-core<2,>=1.0
+mesharc<1,>=0.6
+
+[dev]
+pytest>=8
+pytest-asyncio>=0.23
+langchain-tests==1.1.9
+build
+twine
+ruff>=0.16
+mypy>=1.18

langchain_mesharc-0.1.0/langchain_mesharc.egg-info/top_level.txt

@@ -0,0 +1 @@
+langchain_mesharc

langchain_mesharc-0.1.0/pyproject.toml

@@ -0,0 +1,58 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "langchain-mesharc"
+version = "0.1.0"
+description = "LangChain integration for MeshArc: web pages as Documents, and web search and page reading as agent tools."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+dependencies = ["langchain-core>=1.0,<2", "mesharc>=0.6,<1"]
+
+keywords = ["langchain", "mesharc", "web-scraping", "crawler", "document-loader", "rag", "markdown", "change-detection"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Typing :: Typed",
+  "Operating System :: OS Independent",
+  "Topic :: Internet :: WWW/HTTP",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "pytest-asyncio>=0.23", "langchain-tests==1.1.9", "build", "twine", "ruff>=0.16", "mypy>=1.18"]
+
+[project.urls]
+Homepage = "https://mesharc.dev"
+Documentation = "https://github.com/mesharc-org/langchain-mesharc#readme"
+Source = "https://github.com/mesharc-org/langchain-mesharc"
+Issues = "https://github.com/mesharc-org/langchain-mesharc/issues"
+Privacy = "https://mesharc.dev/legal/privacy"
+
+[tool.setuptools.packages.find]
+include = ["langchain_mesharc*"]
+
+[tool.setuptools.package-data]
+langchain_mesharc = ["py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+
+[tool.ruff]
+target-version = "py310"
+extend-exclude = [".venv*"]
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F"]
+
+[tool.mypy]
+python_version = "3.10"
+files = ["langchain_mesharc"]

langchain_mesharc-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+