mdengine 0.1.0__py3-none-any.whl

md_generator/__init__.py

@@ -0,0 +1,5 @@
+"""md_generator: PDF, Office, image, text, and ZIP to Markdown conversion."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

md_generator/archive/__init__.py

@@ -0,0 +1 @@
+"""zip-to-md: ZIP archive to Markdown + assets."""

md_generator/archive/api/__init__.py

@@ -0,0 +1 @@
+"""HTTP API and MCP entrypoints for zip-to-md."""

md_generator/archive/api/convert_runner.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import io
+import zipfile
+from pathlib import Path
+
+from md_generator.archive.convert_impl import convert_zip
+from md_generator.archive.options import ConvertOptions
+
+
+def build_artifact_zip_bytes(zip_path: Path, options: ConvertOptions) -> bytes:
+    """Run conversion into a temp artifact dir; return ZIP bytes (document.md + assets/)."""
+    import tempfile
+
+    buf = io.BytesIO()
+    with tempfile.TemporaryDirectory() as td:
+        out = Path(td) / "artifact"
+        convert_zip(zip_path, out, options)
+        with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+            doc = out / "document.md"
+            if doc.is_file():
+                zf.write(doc, "document.md")
+            assets = out / "assets"
+            if assets.is_dir():
+                for p in sorted(assets.rglob("*")):
+                    if p.is_file():
+                        arc = Path("assets") / p.relative_to(assets)
+                        zf.write(p, arc.as_posix())
+    return buf.getvalue()

md_generator/archive/api/jobs.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import shutil
+import threading
+import time
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable
+
+
+@dataclass
+class Job:
+    job_id: str
+    workspace: Path
+    status: str = "queued"
+    error: str | None = None
+    created_at: float = field(default_factory=time.time)
+    zip_path: Path | None = None
+
+
+class JobStore:
+    def __init__(self, base_temp: Path | None, ttl_seconds: int) -> None:
+        self._base = base_temp
+        self._ttl = ttl_seconds
+        self._jobs: dict[str, Job] = {}
+        self._lock = threading.Lock()
+        self._sweeper_started = False
+
+    def _root(self) -> Path:
+        import tempfile
+
+        if self._base:
+            p = Path(self._base)
+            p.mkdir(parents=True, exist_ok=True)
+            return p
+        return Path(tempfile.gettempdir()) / "zip-to-md-jobs"
+
+    def start_sweeper(self) -> None:
+        if self._sweeper_started:
+            return
+        self._sweeper_started = True
+
+        def loop() -> None:
+            while True:
+                time.sleep(60)
+                self.sweep()
+
+        t = threading.Thread(target=loop, daemon=True)
+        t.start()
+
+    def sweep(self) -> None:
+        now = time.time()
+        with self._lock:
+            dead: list[str] = []
+            for jid, job in self._jobs.items():
+                if job.status in ("done", "failed") and now - job.created_at > self._ttl:
+                    dead.append(jid)
+            for jid in dead:
+                job = self._jobs.pop(jid, None)
+                if job and job.workspace.exists():
+                    shutil.rmtree(job.workspace, ignore_errors=True)
+
+    def create_job(self) -> Job:
+        jid = str(uuid.uuid4())
+        ws = self._root() / jid
+        ws.mkdir(parents=True, exist_ok=True)
+        job = Job(job_id=jid, workspace=ws)
+        with self._lock:
+            self._jobs[jid] = job
+        return job
+
+    def get(self, job_id: str) -> Job | None:
+        with self._lock:
+            return self._jobs.get(job_id)
+
+    def run_async(self, job: Job, fn: Callable[[], None]) -> None:
+        def target() -> None:
+            try:
+                job.status = "running"
+                fn()
+                job.status = "done"
+            except Exception as e:
+                job.status = "failed"
+                job.error = str(e)
+
+        threading.Thread(target=target, daemon=True).start()
+
+    def remove_after_download(self, job: Job) -> None:
+        with self._lock:
+            self._jobs.pop(job.job_id, None)
+        if job.workspace.exists():
+            shutil.rmtree(job.workspace, ignore_errors=True)

md_generator/archive/api/main.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI, File, HTTPException, Request, UploadFile
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, Response
+from starlette.background import BackgroundTask
+
+from md_generator.archive.api.convert_runner import build_artifact_zip_bytes
+from md_generator.archive.api.jobs import JobStore
+from md_generator.archive.api.mcp_setup import build_mcp_stack
+from md_generator.archive.api.query_options import convert_options_from_query
+from md_generator.archive.api.settings import ApiSettings, cors_list
+from md_generator.archive.options import DEFAULT_IMAGE_TO_MD_ENGINES
+
+_mcp, _mcp_http = build_mcp_stack(mount_under_fastapi=True)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    settings = ApiSettings()
+    base = Path(settings.temp_dir) if settings.temp_dir else None
+    store = JobStore(base, settings.job_ttl_seconds)
+    store.start_sweeper()
+    app.state.settings = settings
+    app.state.job_store = store
+    async with _mcp.session_manager.run():
+        yield
+
+
+app = FastAPI(title="zip-to-md", lifespan=lifespan)
+app.mount("/mcp", _mcp_http)
+
+_bootstrap_settings = ApiSettings()
+_origins = cors_list(_bootstrap_settings)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=_origins,
+    allow_credentials="*" not in _origins,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+def _merge_repo_root(settings: ApiSettings, query_repo: str | None) -> str | None:
+    if query_repo and query_repo.strip():
+        return query_repo.strip()
+    return settings.repo_root
+
+
+async def _read_upload_limited(upload: UploadFile, max_bytes: int) -> bytes:
+    data = bytearray()
+    chunk_size = 1024 * 1024
+    while True:
+        chunk = await upload.read(chunk_size)
+        if not chunk:
+            break
+        data += chunk
+        if len(data) > max_bytes:
+            raise HTTPException(
+                status_code=413,
+                detail="Upload exceeds ZIP_TO_MD_MAX_UPLOAD_MB",
+            )
+    return bytes(data)
+
+
+@app.post("/convert/sync")
+async def convert_sync(
+    request: Request,
+    file: UploadFile = File(...),
+    enable_office: bool = True,
+    image_ocr: bool = False,
+    pdf_ocr: bool = False,
+    max_bytes: int = 512_000,
+    expand_nested_zips: bool = True,
+    max_nested_zip_depth: int = 16,
+    repo_root: str | None = None,
+    use_image_to_md: bool = True,
+    image_to_md_engines: str = DEFAULT_IMAGE_TO_MD_ENGINES,
+    image_to_md_strategy: str = "best",
+    image_to_md_title: str = "",
+) -> Response:
+    settings: ApiSettings = request.app.state.settings
+    if not file.filename or not file.filename.lower().endswith(".zip"):
+        raise HTTPException(400, detail="Expected a .zip file upload (multipart field 'file')")
+    max_u = settings.max_upload_mb * 1024 * 1024
+    max_sync = settings.max_sync_upload_mb * 1024 * 1024
+    body = await _read_upload_limited(file, max_u)
+    if len(body) > max_sync:
+        raise HTTPException(
+            status_code=409,
+            detail="File too large for synchronous conversion; use POST /convert/jobs",
+        )
+    rr = _merge_repo_root(settings, repo_root)
+    opts = convert_options_from_query(
+        enable_office=enable_office,
+        image_ocr=image_ocr,
+        pdf_ocr=pdf_ocr,
+        max_bytes=max_bytes,
+        expand_nested_zips=expand_nested_zips,
+        max_nested_zip_depth=max_nested_zip_depth,
+        repo_root=rr,
+        use_image_to_md=use_image_to_md,
+        image_to_md_engines=image_to_md_engines,
+        image_to_md_strategy=image_to_md_strategy,
+        image_to_md_title=image_to_md_title,
+    )
+    import tempfile
+
+    with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tf:
+        tf.write(body)
+        tmp_in = Path(tf.name)
+    try:
+        zbytes = build_artifact_zip_bytes(tmp_in, opts)
+    finally:
+        tmp_in.unlink(missing_ok=True)
+    return Response(
+        content=zbytes,
+        media_type="application/zip",
+        headers={"Content-Disposition": 'attachment; filename="artifact.zip"'},
+    )
+
+
+@app.post("/convert/jobs")
+async def convert_jobs(
+    request: Request,
+    file: UploadFile = File(...),
+    enable_office: bool = True,
+    image_ocr: bool = False,
+    pdf_ocr: bool = False,
+    max_bytes: int = 512_000,
+    expand_nested_zips: bool = True,
+    max_nested_zip_depth: int = 16,
+    repo_root: str | None = None,
+    use_image_to_md: bool = True,
+    image_to_md_engines: str = DEFAULT_IMAGE_TO_MD_ENGINES,
+    image_to_md_strategy: str = "best",
+    image_to_md_title: str = "",
+) -> dict:
+    settings: ApiSettings = request.app.state.settings
+    store: JobStore = request.app.state.job_store
+    if not file.filename or not file.filename.lower().endswith(".zip"):
+        raise HTTPException(400, detail="Expected a .zip file upload (multipart field 'file')")
+    max_u = settings.max_upload_mb * 1024 * 1024
+    body = await _read_upload_limited(file, max_u)
+    rr = _merge_repo_root(settings, repo_root)
+    opts = convert_options_from_query(
+        enable_office=enable_office,
+        image_ocr=image_ocr,
+        pdf_ocr=pdf_ocr,
+        max_bytes=max_bytes,
+        expand_nested_zips=expand_nested_zips,
+        max_nested_zip_depth=max_nested_zip_depth,
+        repo_root=rr,
+        use_image_to_md=use_image_to_md,
+        image_to_md_engines=image_to_md_engines,
+        image_to_md_strategy=image_to_md_strategy,
+        image_to_md_title=image_to_md_title,
+    )
+    job = store.create_job()
+    inp = job.workspace / "upload.zip"
+    inp.write_bytes(body)
+
+    def work() -> None:
+        zpath = job.workspace / "artifact.zip"
+        data = build_artifact_zip_bytes(inp, opts)
+        zpath.write_bytes(data)
+        job.zip_path = zpath
+
+    store.run_async(job, work)
+    return {"job_id": job.job_id, "status": job.status}
+
+
+@app.get("/convert/jobs/{job_id}")
+async def job_status(request: Request, job_id: str) -> dict:
+    store: JobStore = request.app.state.job_store
+    job = store.get(job_id)
+    if not job:
+        raise HTTPException(404, detail="Unknown job_id")
+    return {
+        "status": job.status,
+        "error": job.error,
+        "created_at": job.created_at,
+    }
+
+
+@app.get("/convert/jobs/{job_id}/download", response_class=FileResponse)
+async def job_download(request: Request, job_id: str) -> FileResponse:
+    store: JobStore = request.app.state.job_store
+    job = store.get(job_id)
+    if not job:
+        raise HTTPException(404, detail="Unknown job_id")
+    if job.status != "done" or not job.zip_path or not job.zip_path.is_file():
+        raise HTTPException(400, detail="Job is not ready for download")
+    path = job.zip_path
+    task = BackgroundTask(store.remove_after_download, job)
+    return FileResponse(
+        path,
+        media_type="application/zip",
+        filename="artifact.zip",
+        background=task,
+    )

md_generator/archive/api/mcp_server.py

@@ -0,0 +1,38 @@
+"""
+Standalone MCP server (stdio, SSE, or streamable-http).
+
+Examples:
+  python -m api.mcp_server
+  python -m api.mcp_server --transport sse
+  python -m api.mcp_server --transport streamable-http
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="zip-to-md MCP server")
+    parser.add_argument(
+        "--transport",
+        choices=("stdio", "sse", "streamable-http"),
+        default="stdio",
+    )
+    args = parser.parse_args()
+
+    from md_generator.archive.api.mcp_setup import build_mcp_stack
+
+    mcp, _ = build_mcp_stack(mount_under_fastapi=False)
+
+    if args.transport == "stdio":
+        asyncio.run(mcp.run_stdio_async())
+    elif args.transport == "sse":
+        asyncio.run(mcp.run_sse_async())
+    else:
+        asyncio.run(mcp.run_streamable_http_async())
+
+
+if __name__ == "__main__":
+    main()

md_generator/archive/api/mcp_setup.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import base64
+import re
+import tempfile
+from pathlib import Path
+
+from mcp.server.fastmcp import FastMCP
+
+from md_generator.archive.api.convert_runner import build_artifact_zip_bytes
+from md_generator.archive.api.query_options import convert_options_from_query
+from md_generator.archive.api.settings import ApiSettings
+from md_generator.archive.options import ConvertOptions
+
+
+def _decode_base64_zip(data: str) -> bytes:
+    s = data.strip()
+    if s.startswith("data:"):
+        parts = s.split(",", 1)
+        if len(parts) == 2:
+            s = parts[1]
+    return base64.b64decode(s, validate=False)
+
+
+def build_mcp_stack(*, mount_under_fastapi: bool = False) -> tuple[FastMCP, object]:
+    path = "/" if mount_under_fastapi else "/mcp"
+    mcp = FastMCP(
+        "zip-to-md",
+        instructions="Convert .zip archives to Markdown artifact ZIP bundles (document.md + assets/).",
+        streamable_http_path=path,
+    )
+    settings = ApiSettings()
+
+    def _opts() -> ConvertOptions:
+        return convert_options_from_query(
+            repo_root=settings.repo_root,
+            use_image_to_md=settings.use_image_to_md,
+            image_to_md_engines=settings.image_to_md_engines,
+            image_to_md_strategy=settings.image_to_md_strategy,
+            image_to_md_title=settings.image_to_md_title,
+        )
+
+    @mcp.tool()
+    def convert_zip_to_artifact_zip(zip_path: str) -> str:
+        """Convert a local .zip path on the server to a temporary artifact.zip path."""
+        src = Path(zip_path).expanduser().resolve()
+        if not src.is_file() or src.suffix.lower() != ".zip":
+            raise ValueError("zip_path must be an existing .zip file")
+        data = build_artifact_zip_bytes(src, _opts())
+        fd, name = tempfile.mkstemp(suffix=".zip", prefix="zip-to-md-artifact-")
+        import os
+
+        os.close(fd)
+        out = Path(name)
+        out.write_bytes(data)
+        return str(out)
+
+    @mcp.tool()
+    def convert_zip_base64_to_artifact_zip(
+        zip_base64: str,
+        filename: str = "upload.zip",
+    ) -> str:
+        """Decode base64 .zip (optional data:...;base64, prefix) and write artifact.zip path."""
+        raw = _decode_base64_zip(zip_base64)
+        max_b = settings.max_upload_mb * 1024 * 1024
+        if len(raw) > max_b:
+            raise ValueError(f"Decoded file exceeds ZIP_TO_MD_MAX_UPLOAD_MB ({settings.max_upload_mb})")
+        safe = re.sub(r"[^\w.\-]+", "_", filename) or "upload.zip"
+        if not safe.lower().endswith(".zip"):
+            safe += ".zip"
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / safe
+            p.write_bytes(raw)
+            data = build_artifact_zip_bytes(p, _opts())
+        fd, name = tempfile.mkstemp(suffix=".zip", prefix="zip-to-md-artifact-")
+        import os
+
+        os.close(fd)
+        out = Path(name)
+        out.write_bytes(data)
+        return str(out)
+
+    sub = mcp.streamable_http_app()
+    return mcp, sub

md_generator/archive/api/query_options.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from md_generator.archive.options import DEFAULT_IMAGE_TO_MD_ENGINES, ConvertOptions
+
+
+def convert_options_from_query(
+    *,
+    enable_office: bool = True,
+    image_ocr: bool = False,
+    pdf_ocr: bool = False,
+    max_bytes: int = 512_000,
+    expand_nested_zips: bool = True,
+    max_nested_zip_depth: int = 16,
+    repo_root: str | None = None,
+    use_image_to_md: bool = True,
+    image_to_md_engines: str = DEFAULT_IMAGE_TO_MD_ENGINES,
+    image_to_md_strategy: str = "best",
+    image_to_md_title: str = "",
+) -> ConvertOptions:
+    """Map API query parameters to ConvertOptions."""
+    return ConvertOptions(
+        artifact_layout=True,
+        verbose=False,
+        enable_office=enable_office,
+        image_ocr=image_ocr,
+        pdf_ocr=pdf_ocr,
+        max_bytes=max_bytes,
+        repo_root=repo_root,
+        expand_nested_zips=expand_nested_zips,
+        max_nested_zip_depth=max_nested_zip_depth,
+        use_image_to_md=use_image_to_md,
+        image_to_md_engines=(image_to_md_engines or DEFAULT_IMAGE_TO_MD_ENGINES).strip()
+        or DEFAULT_IMAGE_TO_MD_ENGINES,
+        image_to_md_strategy=image_to_md_strategy if image_to_md_strategy in ("best", "compare") else "best",
+        image_to_md_title=(image_to_md_title or "").strip(),
+    )

md_generator/archive/api/settings.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from md_generator.archive.options import DEFAULT_IMAGE_TO_MD_ENGINES
+
+
+class ApiSettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="ZIP_TO_MD_",
+        env_file=".env",
+        extra="ignore",
+    )
+
+    max_upload_mb: int = 200
+    max_sync_upload_mb: int = 40
+    job_ttl_seconds: int = 3600
+    temp_dir: str | None = None
+    cors_origins: str = "*"
+    """Deprecated: ignored; converters are imported from the md_generator package."""
+    repo_root: str | None = None
+    use_image_to_md: bool = True
+    image_to_md_engines: str = DEFAULT_IMAGE_TO_MD_ENGINES
+    image_to_md_strategy: str = "best"
+    image_to_md_title: str = ""
+
+
+def cors_list(settings: ApiSettings) -> list[str]:
+    raw = (settings.cors_origins or "*").strip()
+    if raw == "*":
+        return ["*"]
+    return [o.strip() for o in raw.split(",") if o.strip()]