mdengine 0.10.0__tar.gz → 0.11.0__tar.gz

{mdengine-0.10.0 → mdengine-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mdengine
-Version: 0.10.0
+Version: 0.11.0
 Summary: Convert PDF, Office, images, audio, video, text/JSON/XML, ZIP archives, web URLs, databases, graphs, and OpenAPI specs to Markdown.
 Author: mdengine contributors
 License-Expression: MIT
@@ -113,6 +113,11 @@ Requires-Dist: openapi-spec-validator>=0.7.0; extra == "openapi"
 Provides-Extra: codeflow
 Requires-Dist: networkx>=3.2.0; extra == "codeflow"
 Requires-Dist: javalang>=0.13.0; extra == "codeflow"
+Provides-Extra: sap
+Requires-Dist: pyyaml>=6.0.0; extra == "sap"
+Requires-Dist: networkx>=3.2.0; extra == "sap"
+Requires-Dist: pydantic>=2.0.0; extra == "sap"
+Requires-Dist: pydantic-settings>=2.0.0; extra == "sap"
 Provides-Extra: codeflow-worker
 Requires-Dist: celery>=5.3.0; extra == "codeflow-worker"
 Requires-Dist: redis>=5.0.0; extra == "codeflow-worker"

{mdengine-0.10.0 → mdengine-0.11.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mdengine"
-version = "0.10.0"
+version = "0.11.0"
 description = "Convert PDF, Office, images, audio, video, text/JSON/XML, ZIP archives, web URLs, databases, graphs, and OpenAPI specs to Markdown."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -117,6 +117,12 @@ codeflow = [
   "networkx>=3.2.0",
   "javalang>=0.13.0",
 ]
+sap = [
+  "pyyaml>=6.0.0",
+  "networkx>=3.2.0",
+  "pydantic>=2.0.0",
+  "pydantic-settings>=2.0.0",
+]
 # Optional: horizontal workers for codeflow API (Celery + Redis). Default job path uses threads.
 codeflow-worker = [
   "celery>=5.3.0",
@@ -278,6 +284,9 @@ md-codeflow-mcp = "md_generator.codeflow.api.mcp_server:main"
 md-log = "md_generator.log.cli.main:main"
 md-log-api = "md_generator.log.api.run:main"
 md-log-mcp = "md_generator.log.api.mcp_server:main"
+md-sap = "md_generator.sap.cli.main:main"
+md-sap-api = "md_generator.sap.api.run:main"
+md-sap-mcp = "md_generator.sap.api.mcp_server:main"
 md-otel = "md_generator.otel.cli.main:main"
 mdengine = "md_generator.engine_cli:main"
 
@@ -290,6 +299,7 @@ include = ["md_generator*"]
 "md_generator.graph.config" = ["*.yaml"]
 "md_generator.openapi.config" = ["*.yaml"]
 "md_generator.log.config" = ["*.yaml", "presets/*.yaml"]
+"md_generator.sap.config" = ["*.yaml"]
 "md_generator.tools.assistant" = ["data/**/*"]
 
 [tool.pytest.ini_options]
@@ -313,5 +323,6 @@ testpaths = [
   "openapi-to-md/tests",
   "codeflow-to-md/tests",
   "log-to-md/tests",
+  "sap-to-md/tests",
 ]
 markers = ["integration: optional dependency or slower checks"]

{mdengine-0.10.0 → mdengine-0.11.0}/src/md_generator/__init__.py

@@ -1,5 +1,5 @@
 """md_generator: PDF, Office, image, text, and ZIP to Markdown conversion."""
 
-__version__ = "0.10.0"
+__version__ = "0.11.0"
 
 __all__ = ["__version__"]

{mdengine-0.10.0 → mdengine-0.11.0}/src/md_generator/engine_cli.py

@@ -8,7 +8,8 @@ def main(argv: list[str] | None = None) -> int:
     if len(argv) < 1:
         print(
             "Usage: mdengine ai assist … | mdengine ai export … | mdengine skill build … | mdengine db-to-md … | "
-            "mdengine log-to-md … | mdengine graph-to-md … | mdengine openapi-to-md generate … | mdengine codeflow-to-md scan …",
+            "mdengine log-to-md … | mdengine sap-to-md … | mdengine graph-to-md … | mdengine openapi-to-md generate … | "
+            "mdengine codeflow-to-md scan …",
             file=sys.stderr,
         )
         return 2
@@ -73,9 +74,14 @@ def main(argv: list[str] | None = None) -> int:
         from md_generator.codeflow.cli.main import main as cf_main
 
         return cf_main(argv[1:])
+    if argv[0] == "sap-to-md":
+        from md_generator.sap.cli.main import main as sap_main
+
+        return sap_main(argv[1:])
     print(
         "Usage: mdengine ai assist … | mdengine ai export … | mdengine skill build … | mdengine db-to-md … | "
-        "mdengine log-to-md … | mdengine graph-to-md … | mdengine openapi-to-md generate … | mdengine codeflow-to-md scan …",
+        "mdengine log-to-md … | mdengine sap-to-md … | mdengine graph-to-md … | mdengine openapi-to-md generate … | "
+        "mdengine codeflow-to-md scan …",
         file=sys.stderr,
     )
     return 2

mdengine-0.11.0/src/md_generator/sap/__init__.py

@@ -0,0 +1,17 @@
+"""SAP artifact intelligence: parse, analyze, and export AI-ready Markdown."""
+
+from md_generator.sap.core.extractor import extract_to_markdown
+from md_generator.sap.core.run_config import SapRunConfig, load_run_config
+from md_generator.sap.models.entities.kinds import SapObjectKind
+from md_generator.sap.models.entities.sap_object import SapObject
+
+__all__ = [
+    "__version__",
+    "extract_to_markdown",
+    "SapRunConfig",
+    "load_run_config",
+    "SapObject",
+    "SapObjectKind",
+]
+
+__version__ = "0.1.0"

mdengine-0.11.0/src/md_generator/sap/analyzer/authorization/extractor.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import Any
+
+from md_generator.sap.models.entities.kinds import SapObjectKind
+from md_generator.sap.models.entities.sap_object import SapObject
+
+
+def extract_authorization(objects: list[SapObject]) -> list[dict[str, Any]]:
+    checks: list[dict[str, Any]] = []
+    for obj in objects:
+        meta = obj.raw_metadata or {}
+        if obj.kind != SapObjectKind.PROGRAM or "abap" not in meta:
+            continue
+        for a in meta["abap"].get("auth_checks", []):
+            checks.append(
+                {
+                    "program": obj.name,
+                    "object": a.get("object"),
+                    "fields": a.get("fields", []),
+                    "line": a.get("line"),
+                }
+            )
+    return checks

mdengine-0.11.0/src/md_generator/sap/analyzer/governance/classifier.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+PII_PATTERNS = [
+    (re.compile(r"EMAIL", re.I), "PII"),
+    (re.compile(r"\b(SSN|SOCIAL.?SECURITY)\b", re.I), "PII"),
+    (re.compile(r"\b(PHONE|MOBILE|TELNR)\b", re.I), "PII"),
+    (re.compile(r"\b(NAME|FIRSTNAME|LASTNAME|NACHN|VORNA)\b", re.I), "PII"),
+]
+FINANCIAL_PATTERNS = [
+    (re.compile(r"\b(AMOUNT|BETRW|WRBTR|DMBTR|NETWR)\b", re.I), "FINANCIAL"),
+    (re.compile(r"\b(IBAN|BANK|BANKL|BANKN)\b", re.I), "FINANCIAL"),
+    (re.compile(r"\b(TAX|STCEG|MWSKZ)\b", re.I), "FINANCIAL"),
+]
+HR_PATTERNS = [
+    (re.compile(r"\b(SALARY|LGART|BETRG)\b", re.I), "HR_SENSITIVE"),
+    (re.compile(r"^PA\d", re.I), "HR_TABLE"),
+]
+
+
+def classify_field(field_name: str, *, table_name: str = "", domain: str = "") -> str | None:
+    for pat, label in PII_PATTERNS + FINANCIAL_PATTERNS + HR_PATTERNS:
+        if pat.search(field_name) or pat.search(domain):
+            return label
+    if table_name.upper().startswith("PA"):
+        return "HR_TABLE"
+    if table_name.upper() in ("BKPF", "BSEG", "BSID", "BSAD"):
+        return "FINANCIAL_TABLE"
+    return None
+
+
+def classify_objects(objects: list[Any]) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    for obj in objects:
+        meta = getattr(obj, "raw_metadata", {}) or {}
+        ddic = meta.get("ddic")
+        if isinstance(ddic, dict):
+            for f in ddic.get("fields", []):
+                fname = f.get("name", "")
+                cls = classify_field(fname, table_name=obj.name, domain=f.get("domain", ""))
+                if cls:
+                    out.append(
+                        {
+                            "field": fname,
+                            "table": obj.name,
+                            "classification": cls,
+                            "object_id": obj.object_id,
+                        }
+                    )
+    return out

mdengine-0.11.0/src/md_generator/sap/analyzer/lineage/builder.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Any
+
+import networkx as nx
+
+from md_generator.sap.models.entities.sap_object import SapObject
+
+
+def build_lineage_metadata(
+    objects: list[SapObject],
+    graph: nx.MultiDiGraph | None,
+) -> dict[str, list[dict[str, Any]]]:
+    lineage: dict[str, list[dict[str, Any]]] = {}
+    for obj in objects:
+        oid = obj.object_id
+        entry: dict[str, Any] = {
+            "object_id": oid,
+            "name": obj.name,
+            "kind": obj.kind.value,
+            "source_path": str(obj.source_path) if obj.source_path else None,
+            "upstream": [],
+            "downstream": [],
+        }
+        if graph and oid in graph:
+            for pred in graph.predecessors(oid):
+                entry["upstream"].append({"id": pred, "relations": _edge_types(graph, pred, oid)})
+            for succ in graph.successors(oid):
+                entry["downstream"].append({"id": succ, "relations": _edge_types(graph, oid, succ)})
+        lineage[oid] = [entry]
+    return lineage
+
+
+def _edge_types(g: nx.MultiDiGraph, u: str, v: str) -> list[str]:
+    types: list[str] = []
+    if g.has_edge(u, v):
+        for _, data in g[u][v].items():
+            types.append(data.get("relation", "LINK"))
+    return types

mdengine-0.11.0/src/md_generator/sap/analyzer/relationships/engine.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any
+
+import networkx as nx
+
+from md_generator.sap.models.entities.sap_object import SapObject
+
+
+def summarize_relationships(
+    graph: nx.MultiDiGraph,
+    objects: list[SapObject],
+    *,
+    cap: int = 80,
+) -> dict[str, list[dict[str, Any]]]:
+    by_id = {o.object_id: o for o in objects}
+    out: dict[str, list[dict[str, Any]]] = {}
+    for obj in objects:
+        oid = obj.object_id
+        rels: list[dict[str, Any]] = []
+        if oid not in graph:
+            out[oid] = rels
+            continue
+        for _, tgt, data in graph.out_edges(oid, data=True):
+            rels.append(
+                {
+                    "target_id": tgt,
+                    "target_name": by_id[tgt].name if tgt in by_id else tgt,
+                    "relation": data.get("relation", "LINK"),
+                }
+            )
+            if len(rels) >= cap:
+                break
+        out[oid] = rels
+    return out

mdengine-0.11.0/src/md_generator/sap/analyzer/semantics/entity_mapper.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+import re
+
+_PREFIX_RE = re.compile(r"^[ZY]_?", re.I)
+_CAMEL_RE = re.compile(r"([a-z])([A-Z])")
+
+
+def infer_semantic_entity(view_name: str, annotations: dict[str, str] | None = None) -> str:
+    if annotations:
+        for k, v in annotations.items():
+            if "businessobject" in k.lower() or "semantics.businessobject" in k.lower():
+                return v.strip().strip("'\"")
+    name = _PREFIX_RE.sub("", view_name)
+    name = name.replace("_", " ")
+    name = _CAMEL_RE.sub(r"\1 \2", name)
+    parts = [p for p in name.split() if p and p.upper() not in ("I", "C", "VIEW", "DDIC")]
+    if parts:
+        return " ".join(p.capitalize() for p in parts)
+    return view_name

mdengine-0.11.0/src/md_generator/sap/analyzer/validation/extractor.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any
+
+from md_generator.sap.models.entities.kinds import SapObjectKind
+from md_generator.sap.models.entities.sap_object import SapObject
+
+
+def extract_validations(objects: list[SapObject]) -> list[dict[str, Any]]:
+    rules: list[dict[str, Any]] = []
+    for obj in objects:
+        meta = obj.raw_metadata or {}
+        if obj.kind == SapObjectKind.PROGRAM and "abap" in meta:
+            for v in meta["abap"].get("validations", []):
+                rules.append(
+                    {
+                        "source": obj.name,
+                        "source_kind": obj.kind.value,
+                        "rule_type": v.get("rule_type"),
+                        "expression": v.get("expression"),
+                        "line": v.get("line"),
+                    }
+                )
+        if obj.kind == SapObjectKind.CDS_VIEW and "cds" in meta:
+            for k, val in meta["cds"].get("annotations", {}).items():
+                if "assert" in k.lower():
+                    rules.append(
+                        {
+                            "source": obj.name,
+                            "source_kind": obj.kind.value,
+                            "rule_type": "cds_assert",
+                            "expression": f"{k}={val}",
+                        }
+                    )
+    return rules

mdengine-0.11.0/src/md_generator/sap/api/main.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, Response
+from pydantic import ValidationError
+
+from md_generator.sap.api.schemas import SapToMdRunBody
+from md_generator.sap.api.settings import SapApiSettings, cors_list, sqlite_path_resolved
+from md_generator.sap.api.zip_export import build_sap_markdown_zip_bytes
+from md_generator.sap.core.job_manager import SapJobManager
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    settings = SapApiSettings()
+    ws = Path(settings.job_workspace_root) if settings.job_workspace_root else None
+    jobs = SapJobManager(sqlite_path=sqlite_path_resolved(settings), workspace_root=ws)
+    app.state.settings = settings
+    app.state.jobs = jobs
+    yield
+    jobs.close()
+
+
+app = FastAPI(title="sap-to-md", lifespan=lifespan)
+_bootstrap = SapApiSettings()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=cors_list(_bootstrap),
+    allow_credentials="*" not in cors_list(_bootstrap),
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/health")
+async def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.post("/sap-to-md/run")
+async def sap_run_sync(body: SapToMdRunBody) -> Response:
+    try:
+        cfg = body.to_run_config()
+    except ValidationError as e:
+        raise HTTPException(status_code=422, detail=e.errors()) from e
+    if not cfg.input_paths:
+        raise HTTPException(status_code=400, detail="input.paths required")
+    data = build_sap_markdown_zip_bytes(cfg)
+    return Response(
+        content=data,
+        media_type="application/zip",
+        headers={"Content-Disposition": 'attachment; filename="sap-md.zip"'},
+    )
+
+
+@app.post("/sap-to-md/job")
+async def sap_enqueue(body: SapToMdRunBody, request: Request) -> dict[str, str]:
+    cfg = body.to_run_config()
+    if not cfg.input_paths:
+        raise HTTPException(status_code=400, detail="input.paths required")
+    jobs: SapJobManager = request.app.state.jobs
+    job_id = jobs.enqueue(cfg)
+    return {"job_id": job_id}
+
+
+@app.get("/sap-to-md/job/{job_id}")
+async def sap_job_status(job_id: str, request: Request) -> dict:
+    jobs: SapJobManager = request.app.state.jobs
+    rec = jobs.get(job_id)
+    if not rec:
+        raise HTTPException(status_code=404, detail="job not found")
+    return rec.to_api_dict()
+
+
+@app.get("/sap-to-md/job/{job_id}/download")
+async def sap_job_download(job_id: str, request: Request) -> FileResponse:
+    jobs: SapJobManager = request.app.state.jobs
+    rec = jobs.get(job_id)
+    if not rec or rec.status != "COMPLETED" or not rec.zip_path:
+        raise HTTPException(status_code=404, detail="zip not ready")
+    return FileResponse(rec.zip_path, filename="sap-md.zip", media_type="application/zip")

mdengine-0.11.0/src/md_generator/sap/api/mcp_server.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+import os
+
+
+def main() -> None:
+    """Run sap-to-md API (MCP mount can be added when mdengine[mcp] is installed)."""
+    import uvicorn
+
+    host = os.environ.get("SAP_TO_MD_HOST", "127.0.0.1")
+    port = int(os.environ.get("SAP_TO_MD_PORT", "8020"))
+    uvicorn.run("md_generator.sap.api.main:app", host=host, port=port, reload=False)
+
+
+if __name__ == "__main__":
+    main()

mdengine-0.11.0/src/md_generator/sap/api/run.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import os
+
+
+def main() -> None:
+    import uvicorn
+
+    host = os.environ.get("SAP_TO_MD_HOST", "127.0.0.1")
+    port = int(os.environ.get("SAP_TO_MD_PORT", "8020"))
+    uvicorn.run("md_generator.sap.api.main:app", host=host, port=port, reload=False)
+
+
+if __name__ == "__main__":
+    main()

mdengine-0.11.0/src/md_generator/sap/api/schemas.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field, model_validator
+
+from md_generator.sap.core.features import FEATURES
+from md_generator.sap.core.run_config import SapRunConfig
+
+
+class InputSection(BaseModel):
+    paths: list[str] = Field(default_factory=list)
+
+
+class OutputSection(BaseModel):
+    path: str = "output/sap-md"
+    split_files: bool = True
+    write_manifest: bool = True
+    markdown_cross_links: bool = True
+
+
+class FeaturesSection(BaseModel):
+    include: list[str] | None = None
+    exclude: list[str] = Field(default_factory=list)
+
+
+class SapToMdRunBody(BaseModel):
+    input: InputSection = Field(default_factory=InputSection)
+    output: OutputSection = Field(default_factory=OutputSection)
+    features: FeaturesSection = Field(default_factory=FeaturesSection)
+    chunking: dict[str, Any] = Field(default_factory=dict)
+    graph: dict[str, Any] = Field(default_factory=dict)
+    performance: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def check_features(self) -> SapToMdRunBody:
+        if self.features.include:
+            bad = set(self.features.include) - FEATURES
+            if bad:
+                raise ValueError(f"Unknown features: {sorted(bad)}")
+        return self
+
+    def to_run_config(self) -> SapRunConfig:
+        from md_generator.sap.core.run_config import load_run_config
+
+        return load_run_config(None, self.model_dump()).normalized()

mdengine-0.11.0/src/md_generator/sap/api/settings.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class SapApiSettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="SAP_TO_MD_",
+        env_file=".env",
+        extra="ignore",
+    )
+
+    job_sqlite_path: str | None = None
+    job_workspace_root: str | None = None
+    cors_origins: str = "*"
+    max_sync_zip_mb: int = 80
+
+
+def cors_list(settings: SapApiSettings) -> list[str]:
+    raw = (settings.cors_origins or "*").strip()
+    if raw == "*":
+        return ["*"]
+    return [o.strip() for o in raw.split(",") if o.strip()]
+
+
+def sqlite_path_resolved(settings: SapApiSettings) -> Path | None:
+    if settings.job_sqlite_path:
+        return Path(settings.job_sqlite_path)
+    return None

mdengine-0.11.0/src/md_generator/sap/api/zip_export.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import tempfile
+import zipfile
+from pathlib import Path
+
+from md_generator.sap.core.extractor import extract_to_markdown
+from md_generator.sap.core.run_config import SapRunConfig
+
+
+def build_sap_markdown_zip_bytes(cfg: SapRunConfig) -> bytes:
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp) / "out"
+        cfg = cfg.with_output(root)
+        extract_to_markdown(cfg)
+        buf = tempfile.NamedTemporaryFile(delete=False, suffix=".zip")
+        buf.close()
+        zpath = Path(buf.name)
+        with zipfile.ZipFile(zpath, "w", zipfile.ZIP_DEFLATED) as zf:
+            for p in sorted(root.rglob("*")):
+                if p.is_file():
+                    zf.write(p, p.relative_to(root).as_posix())
+        data = zpath.read_bytes()
+        zpath.unlink(missing_ok=True)
+        return data