d365fo-agent-developer 0.6.0__py3-none-any.whl

d365fo_agent/__init__.py

@@ -0,0 +1,2 @@
+"""Local tooling for D365 F&O corpus curation and metadata extraction."""
+

d365fo_agent/aot_relations.py

@@ -0,0 +1,147 @@
+"""Extract AOT table relations — the D365 equivalent of foreign keys.
+
+AX business tables define no SQL foreign keys; the relational graph lives in the AOT
+metadata: every ``AxTable`` (and ``AxTableExtension``) XML carries a ``<Relations>`` block
+with the related table, the relationship type (Association/Composition), both cardinalities,
+and the EXACT join fields (``<AxTableRelationConstraintField>``: Field ↔ RelatedField, plus
+fixed-value constraints). This module walks one or more corpus roots (PackagesLocalDirectory
+and/or editable source trees), parses those blocks, and stores them next to the SQL data
+model so ``find_relations``/``get_sql_model`` can explain table relationships with the same
+authority a foreign key would. Standard library only.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Iterator
+
+_XSI_TYPE = "{http://www.w3.org/2001/XMLSchema-instance}type"
+# Mirrors index_store._NON_AOT_DIRS: build outputs that contain compiled XML copies.
+_SKIP_DIRS = {"bin", "Descriptor", "Resources", "XppMetadata", "AdditionalFiles", "obj"}
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS aot_relations(
+    table_name TEXT, relation_name TEXT, related_table TEXT,
+    relationship_type TEXT, cardinality TEXT, related_cardinality TEXT,
+    edt_relation INTEGER, source_element TEXT,
+    PRIMARY KEY(table_name, relation_name, source_element));
+CREATE TABLE IF NOT EXISTS aot_relation_fields(
+    table_name TEXT, relation_name TEXT, kind TEXT,
+    field TEXT, related_field TEXT, fixed_value TEXT, source_edt TEXT);
+CREATE INDEX IF NOT EXISTS ix_aot_rel_table ON aot_relations(table_name);
+CREATE INDEX IF NOT EXISTS ix_aot_rel_related ON aot_relations(related_table);
+"""
+
+
+def _text(node: ET.Element | None) -> str | None:
+    return node.text.strip() if node is not None and node.text else None
+
+
+def _iter_table_files(root: Path) -> Iterator[tuple[Path, str]]:
+    """Yield (xml_path, element_kind) for every AxTable/AxTableExtension under a corpus root."""
+    for kind in ("AxTable", "AxTableExtension"):
+        for type_dir in root.rglob(kind):
+            if not type_dir.is_dir() or type_dir.name != kind:
+                continue
+            if any(part in _SKIP_DIRS for part in type_dir.relative_to(root).parts):
+                continue
+            for xml_file in type_dir.glob("*.xml"):
+                yield xml_file, kind
+
+
+def parse_table_relations(xml_path: Path, kind: str) -> tuple[str, str, list[dict[str, object]]]:
+    """Parse one AxTable/AxTableExtension XML. Returns (table_name, source_element, relations).
+
+    For an extension ``CustTable.MyModel``, the relations belong to ``CustTable``.
+    """
+    tree = ET.parse(xml_path)
+    root = tree.getroot()
+    element_name = _text(root.find("Name")) or xml_path.stem
+    table_name = element_name.split(".")[0] if kind == "AxTableExtension" else element_name
+
+    relations: list[dict[str, object]] = []
+    for rel in root.iter("AxTableRelation"):
+        constraints: list[dict[str, object]] = []
+        for constraint in rel.iter("AxTableRelationConstraint"):
+            ctype = (constraint.get(_XSI_TYPE) or "AxTableRelationConstraintField")
+            constraints.append({
+                "kind": ctype.replace("AxTableRelationConstraint", "").lower() or "field",
+                "field": _text(constraint.find("Field")),
+                "related_field": _text(constraint.find("RelatedField")),
+                "fixed_value": _text(constraint.find("Value")),
+                "source_edt": _text(constraint.find("SourceEDT")),
+            })
+        relations.append({
+            "name": _text(rel.find("Name")),
+            "related_table": _text(rel.find("RelatedTable")),
+            "relationship_type": _text(rel.find("RelationshipType")),
+            "cardinality": _text(rel.find("Cardinality")),
+            "related_cardinality": _text(rel.find("RelatedTableCardinality")),
+            "edt_relation": 1 if _text(rel.find("EDTRelation")) == "Yes" else 0,
+            "constraints": constraints,
+        })
+    return table_name, element_name, relations
+
+
+def extract_aot_relations(
+    roots: "list[str | Path]",
+    db_path: str | Path,
+    *,
+    progress: "callable[[str, int], None] | None" = None,
+) -> dict[str, int]:
+    """Walk every AxTable/AxTableExtension under ``roots`` and persist their relations."""
+    conn = sqlite3.connect(Path(db_path))
+    conn.executescript("DELETE FROM aot_relations; DELETE FROM aot_relation_fields;"
+                       if conn.execute("SELECT 1 FROM sqlite_master WHERE name='aot_relations'").fetchone()
+                       else "SELECT 1;")
+    conn.executescript(_SCHEMA)
+
+    files = relations_count = 0
+    rel_rows: list[tuple] = []
+    field_rows: list[tuple] = []
+    for root in roots:
+        root = Path(root)
+        for xml_file, kind in _iter_table_files(root):
+            try:
+                table, element, relations = parse_table_relations(xml_file, kind)
+            except OSError:
+                # Windows MAX_PATH: deep AOT paths exceed 260 chars — retry extended-length.
+                try:
+                    table, element, relations = parse_table_relations(
+                        Path("\\\\?\\" + str(xml_file.resolve())), kind)
+                except (OSError, ET.ParseError):
+                    continue
+            except ET.ParseError:
+                continue
+            files += 1
+            for rel in relations:
+                if not rel["related_table"]:
+                    continue
+                relations_count += 1
+                rel_rows.append((table, rel["name"], rel["related_table"],
+                                 rel["relationship_type"], rel["cardinality"],
+                                 rel["related_cardinality"], rel["edt_relation"], element))
+                for c in rel["constraints"]:
+                    field_rows.append((table, rel["name"], c["kind"], c["field"],
+                                       c["related_field"], c["fixed_value"], c["source_edt"]))
+            if len(rel_rows) >= 5000:
+                conn.executemany("INSERT OR REPLACE INTO aot_relations VALUES(?,?,?,?,?,?,?,?)", rel_rows)
+                conn.executemany("INSERT INTO aot_relation_fields VALUES(?,?,?,?,?,?,?)", field_rows)
+                conn.commit()
+                rel_rows, field_rows = [], []
+                if progress:
+                    progress(str(root), relations_count)
+    conn.executemany("INSERT OR REPLACE INTO aot_relations VALUES(?,?,?,?,?,?,?,?)", rel_rows)
+    conn.executemany("INSERT INTO aot_relation_fields VALUES(?,?,?,?,?,?,?)", field_rows)
+    conn.commit()
+    stats = {
+        "files_parsed": files,
+        "relations": conn.execute("SELECT COUNT(*) FROM aot_relations").fetchone()[0],
+        "constraint_fields": conn.execute("SELECT COUNT(*) FROM aot_relation_fields").fetchone()[0],
+        "tables_with_relations": conn.execute(
+            "SELECT COUNT(DISTINCT table_name) FROM aot_relations").fetchone()[0],
+    }
+    conn.close()
+    return stats

d365fo_agent/build.py

@@ -0,0 +1,285 @@
+"""Build / compile adapters — the top rungs of the verification ladder.
+
+Two backends:
+
+* :class:`BuildRunner` plans (or runs) an **MSBuild** of a Visual Studio D365 project. Kept for the
+  full-IDE flow; ``execute=False`` only emits the command.
+* :class:`XppCompiler` drives the standalone **X++ compiler** (``xppc.exe``) directly. This is the
+  one that actually closes the "does it compile?" rung WITHOUT a full AOS/Visual Studio: ``xppc.exe``
+  ships in ``PackagesLocalDirectory/bin`` and compiles a single model against the package metadata,
+  emitting a diagnostics log we parse into structured errors/warnings. It is Windows-only (the
+  compiler is a .NET Framework assembly) and degrades gracefully — :meth:`XppCompiler.available`
+  is False when ``xppc.exe`` is not on this host, so callers report "compile rung needs a Windows
+  D365 host" instead of crashing.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass(slots=True)
+class BuildResult:
+    status: str
+    command: list[str]
+    returncode: int | None = None
+    stdout: str = ""
+    stderr: str = ""
+
+
+class BuildRunner:
+    def __init__(self, msbuild_executable: str = "msbuild.exe") -> None:
+        self.msbuild_executable = msbuild_executable
+
+    def build_project(
+        self,
+        project_path: str | Path,
+        *,
+        execute: bool = False,
+        output_path: str | Path | None = None,
+        additional_properties: dict[str, str] | None = None,
+    ) -> BuildResult:
+        project_path = Path(project_path)
+        properties = dict(additional_properties or {})
+        if output_path is not None:
+            properties["OutputPath"] = str(Path(output_path))
+        command = [self.msbuild_executable, str(project_path)]
+        command.extend(f"/p:{key}={value}" for key, value in properties.items())
+        if not execute:
+            return BuildResult(status="planned", command=command)
+
+        process = subprocess.run(command, capture_output=True, text=True, check=False)
+        return BuildResult(
+            status="succeeded" if process.returncode == 0 else "failed",
+            command=command,
+            returncode=process.returncode,
+            stdout=process.stdout,
+            stderr=process.stderr,
+        )
+
+
+# --- X++ compiler (xppc.exe) --------------------------------------------------------
+
+@dataclass(slots=True)
+class Diagnostic:
+    severity: str  # "error" | "warning"
+    category: str  # e.g. "Compile", "ExternalReference", "Compile Fatal"
+    message: str
+    element: str | None = None  # dynamics://… element path, when present
+    location: str | None = None  # "(line,col),(line,col)", when present
+
+
+@dataclass(slots=True)
+class CompileResult:
+    status: str  # "succeeded" | "failed" | "unavailable"
+    model: str
+    returncode: int | None = None
+    error_count: int = 0
+    warning_count: int = 0
+    diagnostics: list[Diagnostic] = field(default_factory=list)
+    log_path: str | None = None
+    command: list[str] = field(default_factory=list)
+    message: str | None = None
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "status": self.status,
+            "model": self.model,
+            "returncode": self.returncode,
+            "error_count": self.error_count,
+            "warning_count": self.warning_count,
+            "diagnostics": [
+                {"severity": d.severity, "category": d.category, "message": d.message,
+                 "element": d.element, "location": d.location}
+                for d in self.diagnostics
+            ],
+            "log_path": self.log_path,
+            "command": self.command,
+            "message": self.message,
+        }
+
+
+_SUMMARY_RE = re.compile(r"^(Errors|Warnings):\s*(\d+)\s*$")
+# "<Category words> <Severity>: <rest>"  — Severity ∈ {Fatal Error, Error, Warning}
+_DIAG_RE = re.compile(r"^(?P<category>.*?)\b(?P<sev>Fatal Error|Error|Warning):\s*(?P<rest>.*)$")
+_ELEMENT_RE = re.compile(r"(dynamics://\S+?)(?=[:\s]|$)")
+_LOCATION_RE = re.compile(r"\[(\([\d]+,[\d]+\),\([\d]+,[\d]+\))\]")
+
+
+def parse_xppc_log(text: str) -> dict[str, object]:
+    """Parse an ``xppc.exe`` ``-log`` file into structured diagnostics.
+
+    The log groups diagnostics between ``===`` separators, each ``<Category> <Severity>: <msg>``,
+    then a ``Errors: N`` / ``Warnings: N`` summary. Returns
+    ``{error_count, warning_count, diagnostics:[Diagnostic]}``. When the summary lines are absent
+    we fall back to counting parsed diagnostics so callers always get a usable count.
+    """
+    diagnostics: list[Diagnostic] = []
+    summary_errors: int | None = None
+    summary_warnings: int | None = None
+
+    for raw in text.replace("\x00", "").splitlines():
+        line = raw.strip()
+        if not line or set(line) <= {"="}:
+            continue
+        summary = _SUMMARY_RE.match(line)
+        if summary:
+            if summary.group(1) == "Errors":
+                summary_errors = int(summary.group(2))
+            else:
+                summary_warnings = int(summary.group(2))
+            continue
+        diag = _DIAG_RE.match(line)
+        if not diag:
+            continue
+        sev_word = diag.group("sev")
+        rest = diag.group("rest").strip()
+        severity = "error" if "Error" in sev_word else "warning"
+        category = (diag.group("category").strip() + (" Fatal" if sev_word == "Fatal Error" else "")).strip() or "Compile"
+        element_match = _ELEMENT_RE.search(rest)
+        location_match = _LOCATION_RE.search(rest)
+        diagnostics.append(Diagnostic(
+            severity=severity,
+            category=category,
+            message=rest,
+            element=element_match.group(1) if element_match else None,
+            location=location_match.group(1) if location_match else None,
+        ))
+
+    error_count = summary_errors if summary_errors is not None else sum(1 for d in diagnostics if d.severity == "error")
+    warning_count = summary_warnings if summary_warnings is not None else sum(1 for d in diagnostics if d.severity == "warning")
+    return {"error_count": error_count, "warning_count": warning_count, "diagnostics": diagnostics}
+
+
+class XppCompiler:
+    """Drive ``xppc.exe`` to compile a single model against the package metadata."""
+
+    def __init__(self, packages_root: str | Path, xppc_path: str | Path | None = None) -> None:
+        self.packages_root = Path(packages_root)
+        self.xppc_path = Path(xppc_path) if xppc_path else (self.packages_root / "bin" / "xppc.exe")
+
+    def available(self) -> bool:
+        """True iff the X++ compiler is present on this host (it is Windows-only)."""
+        return self.xppc_path.exists()
+
+    def compile_model(
+        self,
+        model: str,
+        *,
+        output_path: str | Path,
+        log_path: str | Path,
+        reference_folder: str | Path | None = None,
+        appchecker: bool = False,
+        xref_file: str | Path | None = None,
+        timeout: int = 1800,
+    ) -> CompileResult:
+        """Compile ``model`` and return structured diagnostics.
+
+        ``appchecker=True`` additionally runs the Appchecker (best-practice) rules. Returns a
+        ``CompileResult`` with ``status="unavailable"`` (not an exception) when ``xppc.exe`` is not
+        on this host, so the caller can report the rung as "needs a Windows D365 host".
+        """
+        if not self.available():
+            return CompileResult(
+                status="unavailable", model=model,
+                message=f"xppc.exe not found at {self.xppc_path} — the compile rung needs a Windows D365 host "
+                        "with PackagesLocalDirectory/bin present.",
+            )
+        output_path = Path(output_path)
+        log_path = Path(log_path)
+        output_path.mkdir(parents=True, exist_ok=True)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        reference_folder = Path(reference_folder) if reference_folder else self.packages_root
+
+        command = [
+            str(self.xppc_path),
+            f"-metadata={self.packages_root}",
+            f"-modelmodule={model}",
+            f"-output={output_path}",
+            f"-referenceFolder={reference_folder}",
+            f"-log={log_path}",
+        ]
+        if appchecker:
+            command.append("-RunAppcheckerRules")
+        if xref_file:
+            command.extend(["-xref", f"-xreffile={Path(xref_file)}"])
+
+        try:
+            process = subprocess.run(command, capture_output=True, text=True, check=False, timeout=timeout)
+        except subprocess.TimeoutExpired:
+            return CompileResult(status="failed", model=model, command=command,
+                                 message=f"xppc.exe timed out after {timeout}s.")
+
+        parsed = {"error_count": 0, "warning_count": 0, "diagnostics": []}
+        if log_path.exists():
+            parsed = parse_xppc_log(log_path.read_text(encoding="utf-8", errors="ignore"))
+        error_count = int(parsed["error_count"])
+        # xppc returns non-zero on failure; treat a clean log + zero exit as success.
+        status = "succeeded" if (process.returncode == 0 and error_count == 0) else "failed"
+        return CompileResult(
+            status=status, model=model, returncode=process.returncode,
+            error_count=error_count, warning_count=int(parsed["warning_count"]),
+            diagnostics=list(parsed["diagnostics"]), log_path=str(log_path), command=command,
+            message=(process.stdout.strip().splitlines()[-1] if process.stdout.strip() else None),
+        )
+
+    def compile_overlay(
+        self,
+        model: str,
+        overlays: list[tuple[str, str]],
+        *,
+        output_path: str | Path,
+        log_path: str | Path,
+        appchecker: bool = False,
+        timeout: int = 1800,
+    ) -> CompileResult:
+        """Compile freshly-generated artifacts IN CONTEXT — closes the generate->compile loop.
+
+        ``overlays`` is a list of ``(relative_path, content)`` where ``relative_path`` is under the
+        PackagesLocalDirectory (e.g. ``BABGeneralLedger/BABGeneralLedger/AxClass/Foo.xml``). Each is
+        temporarily written into the PLD so the model compiles WITH the new artifact, then the PLD is
+        **always restored** (added files removed, overwritten files put back) via try/finally — even
+        on error/timeout. This proves generated X++ actually compiles before it is claimed done.
+        """
+        if not self.available():
+            return CompileResult(
+                status="unavailable", model=model,
+                message=f"xppc.exe not found at {self.xppc_path} — the compile rung needs a Windows D365 host.",
+            )
+        if not overlays:
+            return CompileResult(status="failed", model=model, message="No artifacts to overlay/compile.")
+        backups: list[tuple[Path, bytes | None]] = []
+        try:
+            for relative_path, content in overlays:
+                target = self.packages_root / relative_path
+                target.parent.mkdir(parents=True, exist_ok=True)
+                backups.append((target, target.read_bytes() if target.exists() else None))
+                target.write_text(content, encoding="utf-8")
+                # xppc ALSO (re)writes compiled metadata under <package>/XppMetadata/<model>/... as a
+                # byproduct; snapshot it so the finally block removes/restores it too (it does not yet
+                # exist for a brand-new artifact), leaving the PLD exactly as we found it.
+                meta = self._xppmetadata_path(relative_path)
+                if meta is not None:
+                    backups.append((meta, meta.read_bytes() if meta.exists() else None))
+            result = self.compile_model(model, output_path=output_path, log_path=log_path, timeout=timeout, appchecker=appchecker)
+            result.message = (result.message or "") + f" [compiled with {len(overlays)} overlaid artifact(s), PLD restored]"
+            return result
+        finally:
+            for target, original in reversed(backups):  # restore in reverse order
+                if original is None:
+                    if target.exists():
+                        target.unlink()
+                else:
+                    target.write_bytes(original)
+
+    def _xppmetadata_path(self, relative_path: str) -> Path | None:
+        """The compiled-metadata counterpart xppc writes for a source artifact: a source at
+        ``<package>/<model>/<AxType>/<name>.xml`` maps to ``<package>/XppMetadata/<model>/<AxType>/<name>.xml``."""
+        parts = relative_path.replace("\\", "/").split("/")
+        if len(parts) < 4:
+            return None
+        package, model = parts[0], parts[1]
+        return self.packages_root.joinpath(package, "XppMetadata", model, *parts[2:])