sirdas 0.3.0__tar.gz

sirdas-0.3.0/.gitignore

@@ -0,0 +1,3 @@
+dist/
+__pycache__/
+*.egg-info/

sirdas-0.3.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.5
+Name: sirdas
+Version: 0.3.0
+Summary: Local document parsing for AI agents and ML: PDF, Word, Excel, PowerPoint, HTML, email and scans to Markdown, cited fields, chunks and datasets. Nothing is uploaded.
+Project-URL: Homepage, https://sirdas.app
+Project-URL: Documentation, https://sirdas.app/docs/agents/python.md
+Author: Sırdaş
+License-Expression: MIT
+Keywords: ai-agents,anonymization,dataset,document-parsing,fine-tuning,langchain,llamaindex,llm,local-first,ocr,pdf,rag
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.9
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.2; extra == 'langchain'
+Provides-Extra: llamaindex
+Requires-Dist: llama-index-core>=0.10; extra == 'llamaindex'
+Description-Content-Type: text/markdown
+
+# sirdas (Python)
+
+Local document parsing for AI agents and machine learning. PDF (including scans), Word, Excel, PowerPoint, OpenDocument, EPUB, HTML, CSV, email and images become Markdown, **fields cited with page and position**, tables, form fields, cited chunks and training datasets. **Nothing is uploaded**: it runs on your machine.
+
+```bash
+pip install sirdas          # needs Node.js 20+ (the engine is shared with the CLI and MCP server)
+```
+
+```python
+import sirdas
+
+doc = sirdas.read("factura.pdf")                 # one call does it all
+doc["document_type"]                             # "factura"
+{f["name"]: f["value"] for f in doc["fields"]}   # número, CUFE, NIT, IVA, total… each with page, evidence and bbox
+doc["next_steps"]                                # what to do next
+
+sirdas.convert("informe.docx")["markdown"]
+sirdas.split_documents("lote-escaneado.pdf", output_dir="separados/")
+sirdas.forms("formulario.pdf")                   # filled fields and checkboxes
+sirdas.dataset(["a.pdf", "b.docx"], "ds/", format="alpaca")   # + Hugging Face dataset card
+```
+
+LangChain and LlamaIndex:
+
+```python
+from sirdas.integrations import SirdasLoader, SirdasReader
+docs = SirdasLoader("contrato.pdf").load()              # langchain Documents, stable ids
+nodes = SirdasReader().load_data("historia.pdf")        # llama_index Documents
+```
+
+Personal data is anonymized by default (`anonymize=False` to keep it). Password-protected PDFs raise `PasswordRequired`; pass `password=`.
+
+Docs: https://sirdas.app/docs/agents/python.md

sirdas-0.3.0/README.md

@@ -0,0 +1,33 @@
+# sirdas (Python)
+
+Local document parsing for AI agents and machine learning. PDF (including scans), Word, Excel, PowerPoint, OpenDocument, EPUB, HTML, CSV, email and images become Markdown, **fields cited with page and position**, tables, form fields, cited chunks and training datasets. **Nothing is uploaded**: it runs on your machine.
+
+```bash
+pip install sirdas          # needs Node.js 20+ (the engine is shared with the CLI and MCP server)
+```
+
+```python
+import sirdas
+
+doc = sirdas.read("factura.pdf")                 # one call does it all
+doc["document_type"]                             # "factura"
+{f["name"]: f["value"] for f in doc["fields"]}   # número, CUFE, NIT, IVA, total… each with page, evidence and bbox
+doc["next_steps"]                                # what to do next
+
+sirdas.convert("informe.docx")["markdown"]
+sirdas.split_documents("lote-escaneado.pdf", output_dir="separados/")
+sirdas.forms("formulario.pdf")                   # filled fields and checkboxes
+sirdas.dataset(["a.pdf", "b.docx"], "ds/", format="alpaca")   # + Hugging Face dataset card
+```
+
+LangChain and LlamaIndex:
+
+```python
+from sirdas.integrations import SirdasLoader, SirdasReader
+docs = SirdasLoader("contrato.pdf").load()              # langchain Documents, stable ids
+nodes = SirdasReader().load_data("historia.pdf")        # llama_index Documents
+```
+
+Personal data is anonymized by default (`anonymize=False` to keep it). Password-protected PDFs raise `PasswordRequired`; pass `password=`.
+
+Docs: https://sirdas.app/docs/agents/python.md

sirdas-0.3.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "sirdas"
+version = "0.3.0"
+description = "Local document parsing for AI agents and ML: PDF, Word, Excel, PowerPoint, HTML, email and scans to Markdown, cited fields, chunks and datasets. Nothing is uploaded."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [{ name = "Sırdaş" }]
+keywords = ["pdf", "document-parsing", "rag", "llm", "ai-agents", "ocr", "anonymization", "dataset", "fine-tuning", "langchain", "llamaindex", "local-first"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Topic :: Text Processing",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.optional-dependencies]
+langchain = ["langchain-core>=0.2"]
+llamaindex = ["llama-index-core>=0.10"]
+
+[project.urls]
+Homepage = "https://sirdas.app"
+Documentation = "https://sirdas.app/docs/agents/python.md"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/sirdas"]

sirdas-0.3.0/src/sirdas/__init__.py

@@ -0,0 +1,42 @@
+"""Sırdaş for Python: local document parsing for agents and ML.
+
+Everything runs on this machine through the ``sirdas`` CLI (Node.js >= 20);
+no file, text or field is sent anywhere.
+
+    >>> import sirdas
+    >>> doc = sirdas.read("factura.pdf")
+    >>> doc["document_type"], [f["value"] for f in doc["fields"] if f["name"] == "total"]
+"""
+
+from .client import (
+    SirdasError,
+    PasswordRequired,
+    Sirdas,
+    read,
+    convert,
+    split_documents,
+    forms,
+    classify,
+    chunks,
+    tables,
+    dataset,
+    protect,
+    unlock,
+)
+
+__all__ = [
+    "SirdasError",
+    "PasswordRequired",
+    "Sirdas",
+    "read",
+    "convert",
+    "split_documents",
+    "forms",
+    "classify",
+    "chunks",
+    "tables",
+    "dataset",
+    "protect",
+    "unlock",
+]
+__version__ = "0.3.0"

sirdas-0.3.0/src/sirdas/client.py

@@ -0,0 +1,223 @@
+"""Thin, typed-by-convention wrapper over ``sirdas <command> --json``.
+
+Why a CLI wrapper and not a port: the engine (layout, OCR, anonymization,
+field rules) lives in one TypeScript codebase shared by the web app, the MCP
+server and the CLI. Wrapping it keeps Python results identical to what agents
+get, instead of a second implementation that drifts.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+PathLike = Union[str, "os.PathLike[str]"]
+CLI_VERSION = "0.3.0"
+
+
+class SirdasError(RuntimeError):
+    """The CLI could not process the file. ``code`` is 1 for usage, 2 for the file."""
+
+    def __init__(self, message: str, code: int):
+        super().__init__(message)
+        self.code = code
+
+
+class PasswordRequired(SirdasError):
+    """The PDF needs a password to open (or the one given is wrong)."""
+
+
+def _default_command() -> List[str]:
+    # 1) An explicit binary or script, 2) sirdas on PATH, 3) npx with a pinned version.
+    explicit = os.environ.get("SIRDAS_BIN")
+    if explicit:
+        return ["node", explicit] if explicit.endswith(".js") else [explicit]
+    found = shutil.which("sirdas")
+    if found:
+        return [found]
+    npx = shutil.which("npx")
+    if npx:
+        return [npx, "-y", f"sirdas@{CLI_VERSION}"]
+    raise SirdasError(
+        "Sırdaş needs Node.js 20+. Install it from https://nodejs.org, or set SIRDAS_BIN to the sirdas executable.",
+        1,
+    )
+
+
+class Sirdas:
+    """Client. Reuse one instance; it only resolves the executable once."""
+
+    def __init__(self, command: Optional[Sequence[str]] = None, timeout: Optional[float] = 600):
+        self.command = list(command) if command else _default_command()
+        self.timeout = timeout
+
+    def _run(self, args: Sequence[str], password: Optional[str] = None, parse: bool = True) -> Any:
+        env = dict(os.environ)
+        if password is not None:
+            # Por entorno y no por argumento: los argumentos se ven en `ps`.
+            env["SIRDAS_PASSWORD"] = password
+        proc = subprocess.run(
+            [*self.command, *[str(a) for a in args]],
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            env=env,
+            stdin=subprocess.DEVNULL,
+            timeout=self.timeout,
+        )
+        if proc.returncode != 0:
+            message = (proc.stderr or proc.stdout).strip() or f"sirdas exited with {proc.returncode}"
+            if "contraseña" in message.lower() or "password" in message.lower():
+                raise PasswordRequired(message, proc.returncode)
+            raise SirdasError(message, proc.returncode)
+        return json.loads(proc.stdout) if parse else proc.stdout
+
+    # ---- reading ----
+
+    def read(
+        self,
+        path: PathLike,
+        *,
+        password: Optional[str] = None,
+        anonymize: bool = True,
+        ocr: bool = True,
+        max_tokens: int = 1500,
+    ) -> Dict[str, Any]:
+        """Everything in one call: type, cited fields (page, evidence, bbox), tables,
+        form fields, checkboxes, batch segments, anonymized Markdown, chunks, next steps."""
+        args = ["leer", path, "--json", "--max-tokens", max_tokens]
+        if not anonymize:
+            args.append("--no-anonymize")
+        if not ocr:
+            args.append("--no-ocr")
+        result = self._run(args, password=password)
+        if result.get("status") in ("needs_password", "wrong_password"):
+            raise PasswordRequired(result["status"], 2)
+        # Mismo nombre que devuelve el MCP, para que un ejemplo sirva en los dos.
+        result.setdefault("document_type", (result.get("document") or {}).get("type"))
+        return result
+
+    def convert(self, path: PathLike) -> Dict[str, Any]:
+        """DOCX, XLSX, PPTX, ODT, EPUB, HTML, CSV, EML, RTF… → {format, markdown, tables, warnings}."""
+        return self._run(["convertir", path, "--json"])
+
+    def split_documents(self, path: PathLike, output_dir: Optional[PathLike] = None) -> Dict[str, Any]:
+        """Where each document starts inside a batch PDF; writes one PDF each if ``output_dir``."""
+        args: List[Any] = ["separar", path, "--json"]
+        if output_dir:
+            args += ["-d", output_dir, "--force"]
+        return self._run(args)
+
+    def forms(self, path: PathLike) -> Dict[str, Any]:
+        """Fillable form fields and checkboxes."""
+        return self._run(["formulario", path, "--json"])
+
+    def classify(self, path: PathLike) -> Dict[str, Any]:
+        return self._run(["classify", path, "--json"])
+
+    def chunks(self, path: PathLike, *, max_tokens: int = 1500, anonymize: bool = True) -> List[Dict[str, Any]]:
+        """Cited chunks: stable ``id``, ``pages``, ``section``, ``text``, ``tokens``."""
+        args: List[Any] = ["chunks", path, "--json", "--max-tokens", max_tokens]
+        if not anonymize:
+            args.append("--no-anonymize")
+        return self._run(args)["chunks"]
+
+    def tables(self, path: PathLike) -> List[Dict[str, Any]]:
+        return self._run(["tables", path, "--json"])["tables"]
+
+    # ---- ML ----
+
+    def dataset(
+        self,
+        paths: Sequence[PathLike],
+        output_dir: PathLike,
+        *,
+        format: str = "chat",
+        split: bool = True,
+        anonymize: bool = True,
+        dedupe: bool = True,
+        max_tokens: int = 2000,
+        hugging_face: bool = True,
+    ) -> List[str]:
+        """Write a training/RAG dataset. Formats: text, chat, prompt-completion, alpaca,
+        sharegpt, langchain, llamaindex, embeddings. Returns the written files."""
+        args: List[Any] = ["dataset", *paths, "--format", format, "--max-tokens", max_tokens, "-d", output_dir, "--force"]
+        if split:
+            args.append("--split")
+        if not anonymize:
+            args.append("--no-anonymize")
+        if not dedupe:
+            args.append("--no-dedupe")
+        if hugging_face:
+            args.append("--hf")
+        out = self._run(args, parse=False)
+        return [line for line in out.splitlines() if line.strip() and Path(line.strip()).exists()]
+
+    # ---- security ----
+
+    def protect(self, path: PathLike, password: str, output: PathLike, *, allow_copy: bool = True, allow_print: bool = True) -> Path:
+        args: List[Any] = ["protect", path, "-o", output, "--force"]
+        if not allow_copy:
+            args.append("--no-copy")
+        if not allow_print:
+            args.append("--no-print")
+        self._run(args, password=password, parse=False)
+        return Path(output)
+
+    def unlock(self, path: PathLike, output: PathLike, password: str = "") -> Path:
+        self._run(["unlock", path, "-o", output, "--force"], password=password, parse=False)
+        return Path(output)
+
+
+_default: Optional[Sirdas] = None
+
+
+def _client() -> Sirdas:
+    global _default
+    if _default is None:
+        _default = Sirdas()
+    return _default
+
+
+def read(path: PathLike, **kw: Any) -> Dict[str, Any]:
+    return _client().read(path, **kw)
+
+
+def convert(path: PathLike) -> Dict[str, Any]:
+    return _client().convert(path)
+
+
+def split_documents(path: PathLike, output_dir: Optional[PathLike] = None) -> Dict[str, Any]:
+    return _client().split_documents(path, output_dir)
+
+
+def forms(path: PathLike) -> Dict[str, Any]:
+    return _client().forms(path)
+
+
+def classify(path: PathLike) -> Dict[str, Any]:
+    return _client().classify(path)
+
+
+def chunks(path: PathLike, **kw: Any) -> List[Dict[str, Any]]:
+    return _client().chunks(path, **kw)
+
+
+def tables(path: PathLike) -> List[Dict[str, Any]]:
+    return _client().tables(path)
+
+
+def dataset(paths: Sequence[PathLike], output_dir: PathLike, **kw: Any) -> List[str]:
+    return _client().dataset(paths, output_dir, **kw)
+
+
+def protect(path: PathLike, password: str, output: PathLike, **kw: Any) -> Path:
+    return _client().protect(path, password, output, **kw)
+
+
+def unlock(path: PathLike, output: PathLike, password: str = "") -> Path:
+    return _client().unlock(path, output, password)

sirdas-0.3.0/src/sirdas/integrations.py

@@ -0,0 +1,57 @@
+"""LangChain and LlamaIndex loaders. Imported lazily so neither is a hard dependency."""
+
+from __future__ import annotations
+
+from typing import Any, Iterator, List, Optional
+
+from .client import PathLike, Sirdas
+
+
+def _metadata(doc: dict, chunk: dict, source: str) -> dict:
+    return {
+        "source": source,
+        "id": chunk["id"],
+        "pages": chunk.get("pages"),
+        "section": " › ".join(chunk.get("section") or []),
+        "document_type": doc.get("document_type"),
+        "tokens": chunk.get("tokens"),
+    }
+
+
+class SirdasLoader:
+    """LangChain document loader: one Document per cited chunk.
+
+        from sirdas.integrations import SirdasLoader
+        docs = SirdasLoader("contrato.pdf").load()
+    """
+
+    def __init__(self, path: PathLike, *, anonymize: bool = True, max_tokens: int = 1000, client: Optional[Sirdas] = None):
+        self.path, self.anonymize, self.max_tokens = str(path), anonymize, max_tokens
+        self.client = client or Sirdas()
+
+    def lazy_load(self) -> Iterator[Any]:
+        from langchain_core.documents import Document
+
+        result = self.client.read(self.path, anonymize=self.anonymize, max_tokens=self.max_tokens)
+        for chunk in result.get("chunks", []):
+            yield Document(page_content=chunk["text"], metadata=_metadata(result, chunk, self.path), id=chunk["id"])
+
+    def load(self) -> List[Any]:
+        return list(self.lazy_load())
+
+
+class SirdasReader:
+    """LlamaIndex reader: one TextNode-ready Document per cited chunk, ids stable across runs."""
+
+    def __init__(self, *, anonymize: bool = True, max_tokens: int = 1000, client: Optional[Sirdas] = None):
+        self.anonymize, self.max_tokens = anonymize, max_tokens
+        self.client = client or Sirdas()
+
+    def load_data(self, file: PathLike, extra_info: Optional[dict] = None) -> List[Any]:
+        from llama_index.core import Document
+
+        result = self.client.read(str(file), anonymize=self.anonymize, max_tokens=self.max_tokens)
+        return [
+            Document(text=c["text"], id_=c["id"], metadata={**_metadata(result, c, str(file)), **(extra_info or {})})
+            for c in result.get("chunks", [])
+        ]

sirdas-0.3.0/tests/test_client.py

@@ -0,0 +1,73 @@
+"""Integration tests against the real CLI. Set SIRDAS_BIN to packages/cli/dist/main.js."""
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+import sirdas  # noqa: E402
+
+ROOT = Path(__file__).resolve().parents[3]
+FIXTURES = ROOT / "packages" / "core" / "test" / "fixtures"
+
+
+def make_invoice(path: Path) -> None:
+    script = """
+import { PDFDocument, StandardFonts } from "pdf-lib"; import { writeFileSync } from "node:fs";
+const d = await PDFDocument.create(); const f = await d.embedFont(StandardFonts.Helvetica); const p = d.addPage([595, 842]);
+["FACTURA ELECTRONICA DE VENTA No. FE-77", "CUFE: 1a2b3c4d5e6f", "Cliente: Luis Rojas, C.C. 80.123.456", "Total a pagar: $ 50.000"]
+  .forEach((l, i) => p.drawText(l, { x: 50, y: 780 - i * 22, size: 12, font: f }));
+writeFileSync(process.argv[1], await d.save());
+"""
+    subprocess.run(["node", "--input-type=module", "-e", script, str(path)], check=True, cwd=ROOT)
+
+
+class ClientTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        os.environ.setdefault("SIRDAS_BIN", str(ROOT / "packages" / "cli" / "dist" / "main.js"))
+        cls.tmp = Path(tempfile.mkdtemp())
+        cls.invoice = cls.tmp / "factura.pdf"
+        make_invoice(cls.invoice)
+
+    def test_read_extracts_cited_fields_and_hides_personal_data(self):
+        doc = sirdas.read(self.invoice)
+        self.assertEqual(doc["status"], "ok")
+        self.assertEqual(doc["document_type"], "factura")
+        fields = {f["name"]: f for f in doc["fields"]}
+        self.assertEqual(fields["total"]["value"], "50.000")
+        self.assertEqual(fields["total"]["page"], 1)
+        self.assertNotIn("80.123.456", json.dumps(doc))
+
+    def test_convert_office(self):
+        r = sirdas.convert(FIXTURES / "libro.xlsx")
+        self.assertEqual(r["format"], "xlsx")
+        self.assertIn("FE-1", r["markdown"])
+
+    def test_chunks_have_stable_ids(self):
+        a = sirdas.chunks(self.invoice)
+        b = sirdas.chunks(self.invoice)
+        self.assertTrue(a and a[0]["id"].startswith("fnv1a:"))
+        self.assertEqual([c["id"] for c in a], [c["id"] for c in b])
+
+    def test_dataset_writes_hugging_face_layout(self):
+        out = self.tmp / "ds"
+        files = sirdas.dataset([self.invoice, FIXTURES / "deck.pptx"], out, format="sharegpt", split=False, dedupe=False)
+        names = sorted(Path(f).name for f in files)
+        self.assertIn("README.md", names)
+        self.assertIn("train.jsonl", names)
+
+    def test_password_roundtrip(self):
+        locked = sirdas.protect(self.invoice, "clave", self.tmp / "cerrado.pdf")
+        with self.assertRaises(sirdas.PasswordRequired):
+            sirdas.read(locked)
+        self.assertEqual(sirdas.read(locked, password="clave")["status"], "ok")
+
+
+if __name__ == "__main__":
+    unittest.main()