langchain-flatmark 1.0.1__tar.gz

langchain_flatmark-1.0.1/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.5
+Name: langchain-flatmark
+Version: 1.0.1
+Summary: Document to Markdown API and MCP server for PDF, Word, PowerPoint, Excel and HTML. OCR queue for large files. Hosted in Germany.
+Project-URL: Homepage, https://flatmark.dev
+Project-URL: Documentation, https://github.com/flatmark-dev/flatmark-integrations/blob/main/langchain/README.md
+Project-URL: Repository, https://github.com/flatmark-dev/flatmark-integrations
+Project-URL: Issues, https://github.com/flatmark-dev/flatmark-integrations/issues
+Author-email: podshalocef <contact@podshalocef.com>
+License-Expression: MIT
+Keywords: document-loader,flatmark,langchain,markdown
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: httpx<1,>=0.28
+Requires-Dist: langchain-core<2,>=1.6
+Description-Content-Type: text/markdown
+
+# langchain-flatmark
+
+Document to Markdown API and MCP server for PDF, Word, PowerPoint, Excel and HTML. OCR queue for large files. Hosted in Germany.
+
+A LangChain document loader for the [flatmark API](https://flatmark.dev): `FlatmarkLoader` turns local files and URLs into Markdown `Document`s.
+
+```sh
+pip install langchain-flatmark
+```
+
+```python
+from langchain_flatmark import FlatmarkLoader
+
+docs = FlatmarkLoader("report.pdf").load()
+print(docs[0].page_content)
+```
+
+Direct conversion (`POST /v1/convert`) works without a key at a lower rate limit. [Get an API key](https://flatmark.dev/go/langchain?to=/app/api-keys) and pass it as `api_key=` or set `FLATMARK_API_KEY`.
+
+For large or scanned files, convert through the queue (an API key is required):
+
+```python
+loader = FlatmarkLoader(["scan.pdf", "https://example.com/deck.pptx"], use_queue=True)
+for doc in loader.lazy_load():
+    print(doc.metadata, len(doc.page_content))
+```
+
+The loader submits `POST /v1/convert/jobs`, polls `GET /v1/jobs/{job_id}` until the job's status is `succeeded` or `failed`, then downloads `GET /v1/convert/jobs/{job_id}/result`.
+
+## Documents
+
+One `Document` per source; `page_content` is the Markdown. `metadata` carries `source` (the path or URL as given) plus the `meta` fields of the answer — and `job_id` for a queued conversion.
+
+A URL is downloaded by the loader without your key, then uploaded. Other options: `base_url`, `poll_interval`, `timeout`, and `client` (an `httpx.Client` for proxies or retries).
+
+API reference: https://flatmark.dev/docs · Support: https://flatmark.dev/support · Generated from [`openapi.json`](../openapi.json).

langchain_flatmark-1.0.1/README.md

@@ -0,0 +1,36 @@
+# langchain-flatmark
+
+Document to Markdown API and MCP server for PDF, Word, PowerPoint, Excel and HTML. OCR queue for large files. Hosted in Germany.
+
+A LangChain document loader for the [flatmark API](https://flatmark.dev): `FlatmarkLoader` turns local files and URLs into Markdown `Document`s.
+
+```sh
+pip install langchain-flatmark
+```
+
+```python
+from langchain_flatmark import FlatmarkLoader
+
+docs = FlatmarkLoader("report.pdf").load()
+print(docs[0].page_content)
+```
+
+Direct conversion (`POST /v1/convert`) works without a key at a lower rate limit. [Get an API key](https://flatmark.dev/go/langchain?to=/app/api-keys) and pass it as `api_key=` or set `FLATMARK_API_KEY`.
+
+For large or scanned files, convert through the queue (an API key is required):
+
+```python
+loader = FlatmarkLoader(["scan.pdf", "https://example.com/deck.pptx"], use_queue=True)
+for doc in loader.lazy_load():
+    print(doc.metadata, len(doc.page_content))
+```
+
+The loader submits `POST /v1/convert/jobs`, polls `GET /v1/jobs/{job_id}` until the job's status is `succeeded` or `failed`, then downloads `GET /v1/convert/jobs/{job_id}/result`.
+
+## Documents
+
+One `Document` per source; `page_content` is the Markdown. `metadata` carries `source` (the path or URL as given) plus the `meta` fields of the answer — and `job_id` for a queued conversion.
+
+A URL is downloaded by the loader without your key, then uploaded. Other options: `base_url`, `poll_interval`, `timeout`, and `client` (an `httpx.Client` for proxies or retries).
+
+API reference: https://flatmark.dev/docs · Support: https://flatmark.dev/support · Generated from [`openapi.json`](../openapi.json).

langchain_flatmark-1.0.1/langchain_flatmark/__init__.py

@@ -0,0 +1,5 @@
+"""LangChain integration for flatmark."""
+
+from langchain_flatmark.document_loaders import FlatmarkLoader, FlatmarkError
+
+__all__ = ["FlatmarkLoader", "FlatmarkError"]

langchain_flatmark-1.0.1/langchain_flatmark/document_loaders.py

@@ -0,0 +1,163 @@
+"""flatmark document loader for LangChain.
+
+Generated by listing-sync from the flatmark OpenAPI document; changes here are
+overwritten.
+"""
+
+from __future__ import annotations
+
+import mimetypes
+import os
+import time
+from collections.abc import Iterable, Iterator
+from pathlib import Path, PurePosixPath
+from urllib.parse import quote, urlsplit
+
+import httpx
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+BASE_URL = "https://api.flatmark.dev"
+API_KEY_ENV = "FLATMARK_API_KEY"
+KEY_URL = "https://flatmark.dev/go/langchain?to=/app/api-keys"
+AUTH_HEADER = "X-API-Key"
+USER_AGENT = "langchain-flatmark"
+CONVERT_PATH = "/v1/convert"
+CONVERT_FILE = "file"
+TEXT_FIELD = "markdown"
+META_FIELD = "meta"
+SUBMIT_PATH = "/v1/convert/jobs"
+SUBMIT_FILE = "file"
+POLL_PATH = "/v1/jobs/{job_id}"
+RESULT_PATH = "/v1/convert/jobs/{job_id}/result"
+JOB_ID = "job_id"
+DONE = ('succeeded', 'failed')
+# Standard types older Pythons' mimetypes table lacks (added in 3.13).
+OFFICE_TYPES = {
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+}
+
+
+class FlatmarkError(RuntimeError):
+    """The flatmark API answered with an error, or a queued job failed."""
+
+
+def _media_type(name: str) -> str:
+    guess = mimetypes.guess_type(name)[0]
+    return guess or OFFICE_TYPES.get(Path(name).suffix.lower(), "application/octet-stream")
+
+
+class FlatmarkLoader(BaseLoader):
+    """Load files and URLs as Markdown `Document`s through the flatmark API.
+
+    Each source becomes one `Document` whose `page_content` is the Markdown and
+    whose metadata carries `source` (the path or URL as given).
+
+    Args:
+        file_path: A local path or an http(s) URL, or an iterable of them.
+        api_key: The flatmark API key; defaults to the `FLATMARK_API_KEY`
+            environment variable. Optional for direct conversion (anonymous
+            calls run at a lower rate limit), required with `use_queue=True`.
+        use_queue: Convert through the queue (`POST /v1/convert/jobs`) instead of
+            the direct call (`POST /v1/convert`): the loader submits a job, polls
+            it every `poll_interval` seconds until it finishes or `timeout`
+            seconds pass, and downloads its Markdown.
+        base_url: The API base URL.
+        poll_interval: Seconds between job polls.
+        timeout: Seconds a queued job may take before the loader gives up.
+        client: An `httpx.Client` to send every request with (for proxies,
+            retries or tests); the loader makes and closes its own otherwise.
+    """
+
+    def __init__(
+        self,
+        file_path: str | os.PathLike | Iterable[str | os.PathLike],
+        *,
+        api_key: str | None = None,
+        use_queue: bool = False,
+        base_url: str = BASE_URL,
+        poll_interval: float = 2.0,
+        timeout: float = 600.0,
+        client: httpx.Client | None = None,
+    ) -> None:
+        if isinstance(file_path, (str, os.PathLike)):
+            file_path = [file_path]
+        self.file_paths = [os.fspath(p) for p in file_path]
+        self.api_key = api_key or os.environ.get(API_KEY_ENV) or None
+        if use_queue and not self.api_key:
+            raise ValueError(
+                f"use_queue=True needs an API key: pass api_key= or set "
+                f"{API_KEY_ENV}. Get one at {KEY_URL}"
+            )
+        self.use_queue = use_queue
+        self.base_url = base_url.rstrip("/")
+        self.poll_interval = poll_interval
+        self.timeout = timeout
+        self.client = client
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Convert each source in turn, yielding one `Document` per source."""
+        client = self.client or httpx.Client(timeout=120.0, follow_redirects=True)
+        try:
+            for source in self.file_paths:
+                name, data, media = self._read(client, source)
+                if self.use_queue:
+                    yield self._queued(client, source, name, data, media)
+                else:
+                    yield self._direct(client, source, name, data, media)
+        finally:
+            if self.client is None:
+                client.close()
+
+    def _read(self, client: httpx.Client, source: str) -> tuple[str, bytes, str]:
+        """(file name, bytes, media type) of a path or URL. A URL is fetched
+        without the API key: it goes to someone else's server."""
+        if source.startswith(("http://", "https://")):
+            response = client.get(source, headers={"User-Agent": USER_AGENT})
+            response.raise_for_status()
+            name = PurePosixPath(urlsplit(source).path).name or "document"
+            media = response.headers.get("content-type", "").split(";")[0].strip()
+            if not media or media == "application/octet-stream":
+                media = _media_type(name)
+            return name, response.content, media
+        path = Path(source)
+        return path.name, path.read_bytes(), _media_type(path.name)
+
+    def _call(self, client: httpx.Client, method: str, path: str, **kwargs) -> httpx.Response:
+        headers = {"User-Agent": USER_AGENT}
+        if self.api_key:
+            headers[AUTH_HEADER] = self.api_key
+        response = client.request(method, self.base_url + path, headers=headers, **kwargs)
+        if response.is_error:
+            message = f"{method} {path} answered {response.status_code}: {response.text[:500]}"
+            if response.status_code == 429 and not self.api_key:
+                message += f" (an API key lifts the limit: {KEY_URL})"
+            raise FlatmarkError(message)
+        return response
+
+    def _direct(self, client, source, name, data, media) -> Document:
+        answer = self._call(
+            client, "POST", CONVERT_PATH, files={CONVERT_FILE: (name, data, media)}
+        ).json()
+        meta = (answer.get(META_FIELD) or {}) if META_FIELD else {}
+        return Document(page_content=answer[TEXT_FIELD], metadata={**meta, "source": source})
+
+    def _queued(self, client, source, name, data, media) -> Document:
+        job_id = self._call(
+            client, "POST", SUBMIT_PATH, files={SUBMIT_FILE: (name, data, media)}
+        ).json()[JOB_ID]
+        at = "{" + JOB_ID + "}"
+        deadline = time.monotonic() + self.timeout
+        while True:
+            job = self._call(client, "GET", POLL_PATH.replace(at, quote(job_id, safe=""))).json()
+            if job.get("status") in DONE:
+                break
+            if time.monotonic() >= deadline:
+                raise FlatmarkError(f"job {job_id} not finished after {self.timeout:g} s")
+            time.sleep(self.poll_interval)
+        if job["status"] != DONE[0]:
+            raise FlatmarkError(f"job {job_id} {job['status']}: {job.get('error') or 'no reason given'}")
+        text = self._call(client, "GET", RESULT_PATH.replace(at, quote(job_id, safe=""))).text
+        return Document(page_content=text, metadata={"source": source, JOB_ID: job_id})

langchain_flatmark-1.0.1/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "langchain-flatmark"
+version = "1.0.1"
+description = "Document to Markdown API and MCP server for PDF, Word, PowerPoint, Excel and HTML. OCR queue for large files. Hosted in Germany."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{ name = "podshalocef", email = "contact@podshalocef.com" }]
+keywords = ["langchain", "document-loader", "markdown", "flatmark"]
+classifiers = [
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = ["langchain-core>=1.6,<2", "httpx>=0.28,<1"]
+
+[project.urls]
+Homepage = "https://flatmark.dev"
+Documentation = "https://github.com/flatmark-dev/flatmark-integrations/blob/main/langchain/README.md"
+Repository = "https://github.com/flatmark-dev/flatmark-integrations"
+Issues = "https://github.com/flatmark-dev/flatmark-integrations/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["langchain_flatmark"]

langchain_flatmark-1.0.1/tests/test_document_loaders.py

@@ -0,0 +1,134 @@
+"""Tests for langchain_flatmark, against an httpx.MockTransport (no network)."""
+
+import asyncio
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import httpx
+
+from langchain_flatmark import FlatmarkLoader, FlatmarkError
+from langchain_flatmark.document_loaders import (
+    AUTH_HEADER,
+    BASE_URL,
+    CONVERT_PATH,
+    JOB_ID,
+    META_FIELD,
+    POLL_PATH,
+    RESULT_PATH,
+    SUBMIT_PATH,
+    TEXT_FIELD,
+)
+
+AT = "{" + JOB_ID + "}"
+
+
+class Api:
+    """A fake flatmark API: records requests, answers by path."""
+
+    def __init__(self, statuses=("running", "succeeded"), error=None):
+        self.requests, self.statuses, self.error = [], list(statuses), error
+
+    def __call__(self, request):
+        request.read()
+        self.requests.append(request)
+        path = request.url.path
+        if request.url.host == "files.example.com":
+            return httpx.Response(200, content=b"%PDF-1.7 remote", headers={"content-type": "application/pdf"})
+        if self.error:
+            return httpx.Response(self.error, json={"detail": "nope"})
+        if path.endswith(CONVERT_PATH):
+            answer = {TEXT_FIELD: "# Hello"}
+            if META_FIELD:
+                answer[META_FIELD] = {"filename": "a.pdf", "title": "Hello"}
+            return httpx.Response(200, json=answer)
+        if path.endswith(SUBMIT_PATH):
+            return httpx.Response(202, json={JOB_ID: "j1", "status": "queued"})
+        if path.endswith(POLL_PATH.replace(AT, "j1")):
+            status = self.statuses.pop(0)
+            return httpx.Response(200, json={"id": "j1", "status": status, "error": "too big" if status == "failed" else None})
+        if path.endswith(RESULT_PATH.replace(AT, "j1")):
+            return httpx.Response(200, text="# Queued", headers={"content-type": "text/markdown"})
+        return httpx.Response(404, json={"detail": path})
+
+
+class LoaderTest(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.pdf = Path(tmp.name) / "a.pdf"
+        self.pdf.write_bytes(b"%PDF-1.7 local")
+        env = mock.patch.dict(os.environ, {}, clear=True)
+        env.start()
+        self.addCleanup(env.stop)
+
+    def loader(self, api, source=None, **kwargs):
+        client = httpx.Client(transport=httpx.MockTransport(api))
+        self.addCleanup(client.close)
+        return FlatmarkLoader(source or str(self.pdf), client=client, poll_interval=0, **kwargs)
+
+    def test_direct_convert_is_anonymous_without_a_key(self):
+        api = Api()
+        [doc] = self.loader(api).load()
+        self.assertEqual(doc.page_content, "# Hello")
+        self.assertEqual(doc.metadata["source"], str(self.pdf))
+        [request] = api.requests
+        self.assertEqual(str(request.url), BASE_URL + CONVERT_PATH)
+        self.assertNotIn(AUTH_HEADER.lower(), request.headers)
+        self.assertIn(b'filename="a.pdf"', request.content)
+        self.assertIn(b"Content-Type: application/pdf", request.content)
+
+    def test_key_from_argument_or_environment(self):
+        api = Api()
+        self.loader(api, api_key="k1").load()
+        os.environ["FLATMARK_API_KEY"] = "k2"
+        self.loader(api).load()
+        self.assertEqual([r.headers[AUTH_HEADER] for r in api.requests], ["k1", "k2"])
+
+    def test_url_is_fetched_without_the_key(self):
+        api = Api()
+        url = "https://files.example.com/papers/b.pdf"
+        [doc] = self.loader(api, url, api_key="k1").load()
+        fetch, convert = api.requests
+        self.assertNotIn(AUTH_HEADER.lower(), fetch.headers)
+        self.assertEqual(convert.headers[AUTH_HEADER], "k1")
+        self.assertIn(b'filename="b.pdf"', convert.content)
+        self.assertEqual(doc.metadata["source"], url)
+
+    def test_several_sources_one_document_each(self):
+        docs = self.loader(Api(), [str(self.pdf), self.pdf]).load()
+        self.assertEqual(len(docs), 2)
+
+    def test_queue_submits_polls_and_downloads(self):
+        api = Api()
+        [doc] = self.loader(api, api_key="k1", use_queue=True).load()
+        self.assertEqual(doc.page_content, "# Queued")
+        self.assertEqual(doc.metadata, {"source": str(self.pdf), JOB_ID: "j1"})
+        paths = [r.url.path for r in api.requests]
+        self.assertEqual(
+            paths,
+            [SUBMIT_PATH, POLL_PATH.replace(AT, "j1"), POLL_PATH.replace(AT, "j1"), RESULT_PATH.replace(AT, "j1")],
+        )
+
+    def test_failed_job_raises(self):
+        with self.assertRaisesRegex(FlatmarkError, "too big"):
+            self.loader(Api(statuses=["failed"]), api_key="k1", use_queue=True).load()
+
+    def test_queue_needs_a_key(self):
+        with self.assertRaisesRegex(ValueError, "API key"):
+            FlatmarkLoader(str(self.pdf), use_queue=True)
+
+    def test_api_error_raises_with_status(self):
+        with self.assertRaisesRegex(FlatmarkError, "429"):
+            self.loader(Api(error=429)).load()
+
+    def test_async_load(self):
+        docs = asyncio.run(self.loader(Api()).aload())
+        self.assertEqual(docs[0].page_content, "# Hello")
+
+
+if __name__ == "__main__":
+    unittest.main()