langchain-flatmark 1.0.1__py3-none-any.whl

langchain_flatmark/__init__.py

@@ -0,0 +1,5 @@
+"""LangChain integration for flatmark."""
+
+from langchain_flatmark.document_loaders import FlatmarkLoader, FlatmarkError
+
+__all__ = ["FlatmarkLoader", "FlatmarkError"]

langchain_flatmark/document_loaders.py

@@ -0,0 +1,163 @@
+"""flatmark document loader for LangChain.
+
+Generated by listing-sync from the flatmark OpenAPI document; changes here are
+overwritten.
+"""
+
+from __future__ import annotations
+
+import mimetypes
+import os
+import time
+from collections.abc import Iterable, Iterator
+from pathlib import Path, PurePosixPath
+from urllib.parse import quote, urlsplit
+
+import httpx
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+BASE_URL = "https://api.flatmark.dev"
+API_KEY_ENV = "FLATMARK_API_KEY"
+KEY_URL = "https://flatmark.dev/go/langchain?to=/app/api-keys"
+AUTH_HEADER = "X-API-Key"
+USER_AGENT = "langchain-flatmark"
+CONVERT_PATH = "/v1/convert"
+CONVERT_FILE = "file"
+TEXT_FIELD = "markdown"
+META_FIELD = "meta"
+SUBMIT_PATH = "/v1/convert/jobs"
+SUBMIT_FILE = "file"
+POLL_PATH = "/v1/jobs/{job_id}"
+RESULT_PATH = "/v1/convert/jobs/{job_id}/result"
+JOB_ID = "job_id"
+DONE = ('succeeded', 'failed')
+# Standard types older Pythons' mimetypes table lacks (added in 3.13).
+OFFICE_TYPES = {
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+}
+
+
+class FlatmarkError(RuntimeError):
+    """The flatmark API answered with an error, or a queued job failed."""
+
+
+def _media_type(name: str) -> str:
+    guess = mimetypes.guess_type(name)[0]
+    return guess or OFFICE_TYPES.get(Path(name).suffix.lower(), "application/octet-stream")
+
+
+class FlatmarkLoader(BaseLoader):
+    """Load files and URLs as Markdown `Document`s through the flatmark API.
+
+    Each source becomes one `Document` whose `page_content` is the Markdown and
+    whose metadata carries `source` (the path or URL as given).
+
+    Args:
+        file_path: A local path or an http(s) URL, or an iterable of them.
+        api_key: The flatmark API key; defaults to the `FLATMARK_API_KEY`
+            environment variable. Optional for direct conversion (anonymous
+            calls run at a lower rate limit), required with `use_queue=True`.
+        use_queue: Convert through the queue (`POST /v1/convert/jobs`) instead of
+            the direct call (`POST /v1/convert`): the loader submits a job, polls
+            it every `poll_interval` seconds until it finishes or `timeout`
+            seconds pass, and downloads its Markdown.
+        base_url: The API base URL.
+        poll_interval: Seconds between job polls.
+        timeout: Seconds a queued job may take before the loader gives up.
+        client: An `httpx.Client` to send every request with (for proxies,
+            retries or tests); the loader makes and closes its own otherwise.
+    """
+
+    def __init__(
+        self,
+        file_path: str | os.PathLike | Iterable[str | os.PathLike],
+        *,
+        api_key: str | None = None,
+        use_queue: bool = False,
+        base_url: str = BASE_URL,
+        poll_interval: float = 2.0,
+        timeout: float = 600.0,
+        client: httpx.Client | None = None,
+    ) -> None:
+        if isinstance(file_path, (str, os.PathLike)):
+            file_path = [file_path]
+        self.file_paths = [os.fspath(p) for p in file_path]
+        self.api_key = api_key or os.environ.get(API_KEY_ENV) or None
+        if use_queue and not self.api_key:
+            raise ValueError(
+                f"use_queue=True needs an API key: pass api_key= or set "
+                f"{API_KEY_ENV}. Get one at {KEY_URL}"
+            )
+        self.use_queue = use_queue
+        self.base_url = base_url.rstrip("/")
+        self.poll_interval = poll_interval
+        self.timeout = timeout
+        self.client = client
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Convert each source in turn, yielding one `Document` per source."""
+        client = self.client or httpx.Client(timeout=120.0, follow_redirects=True)
+        try:
+            for source in self.file_paths:
+                name, data, media = self._read(client, source)
+                if self.use_queue:
+                    yield self._queued(client, source, name, data, media)
+                else:
+                    yield self._direct(client, source, name, data, media)
+        finally:
+            if self.client is None:
+                client.close()
+
+    def _read(self, client: httpx.Client, source: str) -> tuple[str, bytes, str]:
+        """(file name, bytes, media type) of a path or URL. A URL is fetched
+        without the API key: it goes to someone else's server."""
+        if source.startswith(("http://", "https://")):
+            response = client.get(source, headers={"User-Agent": USER_AGENT})
+            response.raise_for_status()
+            name = PurePosixPath(urlsplit(source).path).name or "document"
+            media = response.headers.get("content-type", "").split(";")[0].strip()
+            if not media or media == "application/octet-stream":
+                media = _media_type(name)
+            return name, response.content, media
+        path = Path(source)
+        return path.name, path.read_bytes(), _media_type(path.name)
+
+    def _call(self, client: httpx.Client, method: str, path: str, **kwargs) -> httpx.Response:
+        headers = {"User-Agent": USER_AGENT}
+        if self.api_key:
+            headers[AUTH_HEADER] = self.api_key
+        response = client.request(method, self.base_url + path, headers=headers, **kwargs)
+        if response.is_error:
+            message = f"{method} {path} answered {response.status_code}: {response.text[:500]}"
+            if response.status_code == 429 and not self.api_key:
+                message += f" (an API key lifts the limit: {KEY_URL})"
+            raise FlatmarkError(message)
+        return response
+
+    def _direct(self, client, source, name, data, media) -> Document:
+        answer = self._call(
+            client, "POST", CONVERT_PATH, files={CONVERT_FILE: (name, data, media)}
+        ).json()
+        meta = (answer.get(META_FIELD) or {}) if META_FIELD else {}
+        return Document(page_content=answer[TEXT_FIELD], metadata={**meta, "source": source})
+
+    def _queued(self, client, source, name, data, media) -> Document:
+        job_id = self._call(
+            client, "POST", SUBMIT_PATH, files={SUBMIT_FILE: (name, data, media)}
+        ).json()[JOB_ID]
+        at = "{" + JOB_ID + "}"
+        deadline = time.monotonic() + self.timeout
+        while True:
+            job = self._call(client, "GET", POLL_PATH.replace(at, quote(job_id, safe=""))).json()
+            if job.get("status") in DONE:
+                break
+            if time.monotonic() >= deadline:
+                raise FlatmarkError(f"job {job_id} not finished after {self.timeout:g} s")
+            time.sleep(self.poll_interval)
+        if job["status"] != DONE[0]:
+            raise FlatmarkError(f"job {job_id} {job['status']}: {job.get('error') or 'no reason given'}")
+        text = self._call(client, "GET", RESULT_PATH.replace(at, quote(job_id, safe=""))).text
+        return Document(page_content=text, metadata={"source": source, JOB_ID: job_id})

langchain_flatmark-1.0.1.dist-info/METADATA

@@ -0,0 +1,55 @@
+Metadata-Version: 2.5
+Name: langchain-flatmark
+Version: 1.0.1
+Summary: Document to Markdown API and MCP server for PDF, Word, PowerPoint, Excel and HTML. OCR queue for large files. Hosted in Germany.
+Project-URL: Homepage, https://flatmark.dev
+Project-URL: Documentation, https://github.com/flatmark-dev/flatmark-integrations/blob/main/langchain/README.md
+Project-URL: Repository, https://github.com/flatmark-dev/flatmark-integrations
+Project-URL: Issues, https://github.com/flatmark-dev/flatmark-integrations/issues
+Author-email: podshalocef <contact@podshalocef.com>
+License-Expression: MIT
+Keywords: document-loader,flatmark,langchain,markdown
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: httpx<1,>=0.28
+Requires-Dist: langchain-core<2,>=1.6
+Description-Content-Type: text/markdown
+
+# langchain-flatmark
+
+Document to Markdown API and MCP server for PDF, Word, PowerPoint, Excel and HTML. OCR queue for large files. Hosted in Germany.
+
+A LangChain document loader for the [flatmark API](https://flatmark.dev): `FlatmarkLoader` turns local files and URLs into Markdown `Document`s.
+
+```sh
+pip install langchain-flatmark
+```
+
+```python
+from langchain_flatmark import FlatmarkLoader
+
+docs = FlatmarkLoader("report.pdf").load()
+print(docs[0].page_content)
+```
+
+Direct conversion (`POST /v1/convert`) works without a key at a lower rate limit. [Get an API key](https://flatmark.dev/go/langchain?to=/app/api-keys) and pass it as `api_key=` or set `FLATMARK_API_KEY`.
+
+For large or scanned files, convert through the queue (an API key is required):
+
+```python
+loader = FlatmarkLoader(["scan.pdf", "https://example.com/deck.pptx"], use_queue=True)
+for doc in loader.lazy_load():
+    print(doc.metadata, len(doc.page_content))
+```
+
+The loader submits `POST /v1/convert/jobs`, polls `GET /v1/jobs/{job_id}` until the job's status is `succeeded` or `failed`, then downloads `GET /v1/convert/jobs/{job_id}/result`.
+
+## Documents
+
+One `Document` per source; `page_content` is the Markdown. `metadata` carries `source` (the path or URL as given) plus the `meta` fields of the answer — and `job_id` for a queued conversion.
+
+A URL is downloaded by the loader without your key, then uploaded. Other options: `base_url`, `poll_interval`, `timeout`, and `client` (an `httpx.Client` for proxies or retries).
+
+API reference: https://flatmark.dev/docs · Support: https://flatmark.dev/support · Generated from [`openapi.json`](../openapi.json).

langchain_flatmark-1.0.1.dist-info/RECORD

@@ -0,0 +1,6 @@
+langchain_flatmark/__init__.py,sha256=fC_5WRmQA5m9rskV-i-uWnptvaZdb1N4hPCv9xCc12c,168
+langchain_flatmark/document_loaders.py,sha256=bhqBYdWdZlB_TRedcvSZQWKK-1VY-UkmsAwSmx1EuyA,7114
+langchain_flatmark/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+langchain_flatmark-1.0.1.dist-info/METADATA,sha256=4WX6bFl6Jg4jFM3iCsWhs53gMNuefh_bStD7ktbUrjE,2586
+langchain_flatmark-1.0.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+langchain_flatmark-1.0.1.dist-info/RECORD,,

langchain_flatmark-1.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any