md-ddl 0.9.2__py3-none-any.whl

md_ddl/__init__.py

@@ -0,0 +1,118 @@
+"""MD-DDL — the Markdown Data Definition Language standard, packaged for pip.
+
+The distribution carries the standard itself as package data: the
+specification, the agent prompts and skills, the companion guides, the
+reference examples, and the architecture source material. `md-ddl init`
+unpacks that payload into a project so the agents resolve exactly as they do
+when MD-DDL is added as a git submodule.
+
+Spec and repository: https://github.com/Semprini/md-ddl
+"""
+
+from __future__ import annotations
+
+from importlib import resources
+from pathlib import Path
+
+__version__ = "0.9.2"
+
+__all__ = [
+    "DOCS_DIRNAME",
+    "DOCS_PAYLOAD",
+    "WRAPPER_PAYLOAD",
+    "__version__",
+    "docs_sources",
+    "wrapper_source",
+]
+
+#: Directory an installed project keeps the standard in. The Copilot agent
+#: wrappers include `../../.md-ddl/agents/...`, and `md-ddl init` rewrites the
+#: Claude slash commands to match, so this name is part of the contract.
+DOCS_DIRNAME = ".md-ddl"
+
+#: Source-repository directory -> path inside `.md-ddl/` once installed.
+#: `references/industry_standards/` is deliberately absent: the raw BIAN, FHIR
+#: and TM Forum snapshots are 63 MB, and the distilled markdown the agents
+#: actually load ships under `agents/*/skills/standards-alignment/standards/`.
+DOCS_PAYLOAD = {
+    "agents": "agents",
+    "md-ddl-specification": "md-ddl-specification",
+    "guides": "guides",
+    "examples": "examples",
+    "references/architecture": "references/architecture",
+}
+
+#: Source-repository directory -> subdirectory of the packaged `wrappers/`.
+WRAPPER_PAYLOAD = {
+    ".github/agents": "copilot",
+    ".claude/commands": "claude",
+}
+
+
+def _package_dir() -> Path:
+    """Directory this package is installed in."""
+    return Path(str(resources.files(__name__)))
+
+
+def _repo_root() -> Path | None:
+    """Repository root, when running from a source checkout rather than a wheel.
+
+    An editable install or a `python -m md_ddl` from a clone has no packaged
+    payload; the standard lives at the repository root instead. `src/md_ddl`
+    means the root is two levels up.
+    """
+    root = _package_dir().parents[1]
+    return root if (root / "md-ddl-specification").is_dir() else None
+
+
+def docs_sources() -> dict[str, Path]:
+    """Map each `.md-ddl/` destination path to the directory supplying it.
+
+    Raises:
+        FileNotFoundError: if neither the packaged payload nor a source
+            checkout can be found.
+    """
+    package_dir = _package_dir()
+    if (package_dir / "docs").is_dir():
+        return {
+            dest: package_dir / "docs" / dest
+            for dest in DOCS_PAYLOAD.values()
+        }
+
+    repo_root = _repo_root()
+    if repo_root is not None:
+        return {
+            dest: repo_root / source
+            for source, dest in DOCS_PAYLOAD.items()
+        }
+
+    raise FileNotFoundError(
+        f"no MD-DDL payload found in {package_dir} and no source checkout above it; "
+        "reinstall the package with `pip install --force-reinstall md-ddl`"
+    )
+
+
+def wrapper_source(tool: str) -> Path:
+    """Directory holding the agent wrappers for `tool` ('claude' or 'copilot').
+
+    Raises:
+        KeyError: if `tool` is not a known AI tool.
+        FileNotFoundError: if the wrappers cannot be located.
+    """
+    sources = {dest: source for source, dest in WRAPPER_PAYLOAD.items()}
+    if tool not in sources:
+        raise KeyError(tool)
+
+    package_dir = _package_dir()
+    packaged = package_dir / "wrappers" / tool
+    if packaged.is_dir():
+        return packaged
+
+    repo_root = _repo_root()
+    if repo_root is not None:
+        return repo_root / sources[tool]
+
+    raise FileNotFoundError(
+        f"no {tool} agent wrappers found in {package_dir}; "
+        "reinstall the package with `pip install --force-reinstall md-ddl`"
+    )

md_ddl/__main__.py

@@ -0,0 +1,8 @@
+"""Allow `python -m md_ddl` as an alias for the `md-ddl` command."""
+
+import sys
+
+from md_ddl.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

md_ddl/cli.py

@@ -0,0 +1,342 @@
+"""Command-line interface for the packaged MD-DDL standard.
+
+`md-ddl init` is the pip equivalent of adding this repository as a git
+submodule: it unpacks the standard into `.md-ddl/` and installs the agent
+wrappers for Claude Code and/or GitHub Copilot, rewriting their paths so the
+agents resolve against the unpacked copy.
+
+pip has no post-install hook for wheels, so this runs as an explicit command
+after `pip install md-ddl` rather than during the install itself.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import shutil
+import sys
+from pathlib import Path
+
+from md_ddl import DOCS_DIRNAME, __version__, docs_sources, includes, wrapper_source
+
+#: Wrappers that drive review of the standard itself. They belong to the source
+#: repository and are not installed into a consuming project.
+INTERNAL_WRAPPERS = frozenset({"review-md-ddl.agent.md", "review.md"})
+
+#: Claude slash commands carry paths relative to the source repository root.
+#: Retarget them at the unpacked standard, as the bootstrap scripts do.
+CLAUDE_PATH_REWRITES = (
+    (re.compile(r"`agents/"), f"`{DOCS_DIRNAME}/agents/"),
+    (re.compile(r"`md-ddl-specification/"), f"`{DOCS_DIRNAME}/md-ddl-specification/"),
+)
+
+TOOL_TARGETS = {
+    "claude": Path(".claude") / "commands",
+    "copilot": Path(".github") / "agents",
+}
+
+AGENTS = (
+    ("agent-guide", "Agent Guide",
+     "Learning, navigation, concept explanation. Start here."),
+    ("agent-ontology", "Agent Ontology",
+     "Domain discovery, entity modelling, source mapping."),
+    ("agent-artifact", "Agent Artifact",
+     "Physical schema generation (SQL DDL, JSON Schema, Parquet, Cypher)."),
+    ("agent-architect", "Agent Architect",
+     "Data product design, architecture discussion, ODPS manifests."),
+    ("agent-governance", "Agent Governance",
+     "Standards conformance, compliance auditing, governance assurance."),
+)
+
+
+def _agent_table(tool: str) -> str:
+    """Markdown table of the agents, addressed the way `tool` invokes them."""
+    if tool == "claude":
+        rows = [f"`/{name}` | {title} | {purpose}" for name, title, purpose in AGENTS]
+        header = ["Command | Agent | Purpose", "--- | --- | ---"]
+    else:
+        rows = [f"`{name}` | {purpose}" for name, _title, purpose in AGENTS]
+        header = ["Agent | Purpose", "--- | ---"]
+    return "\n".join(header + rows)
+
+
+def _layout_block(project: str, tool: str) -> str:
+    """Aligned project-tree sketch for the instructions file."""
+    wrappers = (
+        (".claude/commands/", "Claude slash commands") if tool == "claude"
+        else (".github/agents/", "Copilot custom agent wrappers")
+    )
+    rows = [
+        (f"{DOCS_DIRNAME}/", "MD-DDL standard (installed — do not edit)"),
+        wrappers,
+        ("domains/", "Domain files (one per business domain)"),
+        ("entities/", "Entity detail files"),
+    ]
+    width = max(len(path) for path, _ in rows)
+    return "\n".join(
+        [f"{project}/"] + [f"  {path:<{width}}  {note}" for path, note in rows]
+    )
+
+
+def _instructions(project: str, tool: str) -> str:
+    """Project-level agent instructions for `tool`."""
+    if tool == "claude":
+        invocation = "Invoke them with slash commands in Claude Code:"
+    else:
+        invocation = "Use the custom agents in `.github/agents/` via GitHub Copilot Chat:"
+    invocation = f"{invocation}\n\n{_agent_table(tool)}"
+
+    return f"""\
+# {project}
+
+This project uses the [MD-DDL standard](https://github.com/Semprini/md-ddl) for
+data domain modelling. MD-DDL is installed from PyPI (`pip install md-ddl`) and
+unpacked into `{DOCS_DIRNAME}/` by `md-ddl init`.
+
+## Agents
+
+MD-DDL provides AI agents for every stage of the modelling lifecycle.
+{invocation}
+
+## Validation
+
+Run the mechanical pre-flight checks with `md-ddl lint <domain-folder>`.
+Everything beyond syntax is agent-driven review — see
+`{DOCS_DIRNAME}/guides/validation-tooling.md`.
+
+## Key references
+
+- MD-DDL specification: `{DOCS_DIRNAME}/md-ddl-specification/`
+- Complete spec (single file): `{DOCS_DIRNAME}/md-ddl-specification/MD-DDL-Complete.md`
+- Examples: `{DOCS_DIRNAME}/examples/`
+
+## Project layout
+
+```
+{_layout_block(project, tool)}
+```
+
+Upgrade the standard with `pip install --upgrade md-ddl && md-ddl init`.
+"""
+
+
+def _guard_submodule(target: Path, force: bool) -> None:
+    """Refuse to overwrite an existing MD-DDL git submodule."""
+    docs_root = target / DOCS_DIRNAME
+    if (docs_root / ".git").exists() and not force:
+        raise SystemExit(
+            f"error: {docs_root} looks like the MD-DDL git submodule.\n"
+            "       Installing over it would discard it. Remove the submodule first\n"
+            "       (git submodule deinit), or re-run with --force to overwrite."
+        )
+
+
+def _install_docs(target: Path) -> list[str]:
+    """Unpack the standard into `<target>/.md-ddl/`. Returns log lines."""
+    docs_root = target / DOCS_DIRNAME
+    log = []
+    docs_root.mkdir(parents=True, exist_ok=True)
+    for dest, source in docs_sources().items():
+        if not source.is_dir():
+            raise SystemExit(f"error: packaged payload is missing {source}")
+        destination = docs_root / dest
+        # Replace rather than merge, so an upgrade cannot leave files that the
+        # new version has deleted.
+        if destination.exists():
+            shutil.rmtree(destination)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copytree(source, destination)
+        log.append(f"  {DOCS_DIRNAME}/{dest}")
+    return log
+
+
+def _install_wrappers(target: Path, tool: str, force: bool) -> tuple[list[str], list[str]]:
+    """Install the agent wrappers for `tool`. Returns (written, skipped)."""
+    source_dir = wrapper_source(tool)
+    dest_dir = target / TOOL_TARGETS[tool]
+    dest_dir.mkdir(parents=True, exist_ok=True)
+
+    written: list[str] = []
+    skipped: list[str] = []
+    for source in sorted(source_dir.glob("*.md")):
+        if source.name in INTERNAL_WRAPPERS:
+            continue
+        destination = dest_dir / source.name
+        if destination.exists() and not force:
+            skipped.append(f"  {TOOL_TARGETS[tool]}/{source.name}")
+            continue
+
+        content = source.read_text(encoding="utf-8")
+        if tool == "claude":
+            for pattern, replacement in CLAUDE_PATH_REWRITES:
+                content = pattern.sub(replacement, content)
+        destination.write_text(content, encoding="utf-8")
+        written.append(f"  {TOOL_TARGETS[tool]}/{source.name}")
+    return written, skipped
+
+
+def _include_roots(target: Path) -> list[Path]:
+    """Everything in a project that may carry `{{INCLUDE}}` directives."""
+    return [target / DOCS_DIRNAME] + [target / path for path in TOOL_TARGETS.values()]
+
+
+def _verify_includes(target: Path) -> int:
+    """Report whether every agent prompt can reach what it includes.
+
+    The wrappers and the unpacked standard use file-relative include paths so
+    they resolve from `.md-ddl/`. Checking after install catches a payload or a
+    wrapper that would otherwise load a silently truncated prompt.
+    """
+    directives = includes.scan(_include_roots(target))
+    broken = [d for d in directives if not d.resolves()]
+
+    if not broken:
+        print(f"\nVerified {len(directives)} include directives resolve.")
+        return 0
+
+    print(f"\nwarning: {len(broken)} of {len(directives)} include directives do not resolve.")
+    print("         The agents referencing them will load an incomplete prompt.")
+    for directive in broken:
+        try:
+            location = directive.file.relative_to(target)
+        except ValueError:
+            location = directive.file
+        print(f"  {location}:{directive.line} -> {directive.target}")
+    return 1
+
+
+def cmd_check(args: argparse.Namespace) -> int:
+    target = Path(args.target).resolve()
+    if not (target / DOCS_DIRNAME).is_dir():
+        raise SystemExit(
+            f"error: no {DOCS_DIRNAME}/ in {target}; run `md-ddl init` there first"
+        )
+    return _verify_includes(target)
+
+
+def cmd_init(args: argparse.Namespace) -> int:
+    target = Path(args.target).resolve()
+    if target.exists() and not target.is_dir():
+        raise SystemExit(f"error: {target} is not a directory")
+    target.mkdir(parents=True, exist_ok=True)
+
+    _guard_submodule(target, args.force)
+
+    tools = ["claude", "copilot"] if args.ai == "both" else [args.ai]
+    project = args.name or target.name
+
+    print(f"Installing MD-DDL {__version__} into {target}")
+
+    print(f"\nStandard ({DOCS_DIRNAME}/):")
+    for line in _install_docs(target):
+        print(line)
+
+    if not args.track:
+        # A self-ignoring directory keeps the unpacked standard out of the
+        # project's history without editing the project's own .gitignore.
+        (target / DOCS_DIRNAME / ".gitignore").write_text(
+            "# Unpacked by `md-ddl init` from the md-ddl PyPI package.\n"
+            "# Re-create it with `md-ddl init` rather than committing it.\n"
+            "*\n",
+            encoding="utf-8",
+        )
+        print(f"  {DOCS_DIRNAME}/.gitignore  (pass --track to commit the standard instead)")
+
+    for tool in tools:
+        written, skipped = _install_wrappers(target, tool, args.force)
+        print(f"\nAgent wrappers ({tool}):")
+        for line in written:
+            print(line)
+        for line in skipped:
+            print(f"{line}  (exists, skipped)")
+
+        if not args.no_instructions:
+            path = target / ("CLAUDE.md" if tool == "claude" else ".github/copilot-instructions.md")
+            if path.exists() and not args.force:
+                print(f"  {path.relative_to(target)}  (exists, skipped)")
+            else:
+                path.parent.mkdir(parents=True, exist_ok=True)
+                path.write_text(_instructions(project, tool), encoding="utf-8")
+                print(f"  {path.relative_to(target)}")
+
+    status = _verify_includes(target)
+
+    print("\nDone. Next steps:")
+    if "claude" in tools:
+        print("  Claude Code:    /agent-guide  — start here")
+    if "copilot" in tools:
+        print("  Copilot Chat:   @agent-guide  — start here")
+    print("  Validate:       md-ddl lint <domain-folder>")
+    return status
+
+
+def cmd_path(args: argparse.Namespace) -> int:
+    """Print where the packaged standard lives, for scripts and CI."""
+    for dest, source in docs_sources().items():
+        print(f"{dest}\t{source}")
+    return 0
+
+
+def run_lint(argv: list[str]) -> int:
+    """Hand the remaining arguments to the linter, untouched."""
+    from md_ddl.lint import main as lint_main
+
+    return lint_main(argv, prog="md-ddl lint")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="md-ddl",
+        description="The MD-DDL standard: project setup and mechanical validation.",
+    )
+    parser.add_argument("--version", action="version", version=f"md-ddl {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    init = subparsers.add_parser(
+        "init",
+        help=f"unpack the standard into {DOCS_DIRNAME}/ and install the agent wrappers",
+    )
+    init.add_argument("target", nargs="?", default=".",
+                      help="project directory to set up (default: current directory)")
+    init.add_argument("--ai", choices=["claude", "copilot", "both"], default="both",
+                      help="which AI tool's agent wrappers to install (default: both)")
+    init.add_argument("--name", help="project name for the generated instructions "
+                                     "(default: the target directory name)")
+    init.add_argument("--force", action="store_true",
+                      help="overwrite existing wrappers and instructions")
+    init.add_argument("--no-instructions", action="store_true",
+                      help="do not write CLAUDE.md or copilot-instructions.md")
+    init.add_argument("--track", action="store_true",
+                      help=f"commit {DOCS_DIRNAME}/ to the project instead of ignoring it")
+    init.set_defaults(func=cmd_init)
+
+    # Registered for `md-ddl --help` only; `main` routes `lint` before parsing so
+    # every linter flag, `--help` included, reaches the linter unchanged.
+    subparsers.add_parser(
+        "lint",
+        help="run the MD-DDL pre-flight checks (see `md-ddl lint --help`)",
+        add_help=False,
+    ).add_argument("lint_args", nargs="*", help=argparse.SUPPRESS)
+
+    check = subparsers.add_parser(
+        "check", help="verify the installed agent prompts resolve what they include")
+    check.add_argument("target", nargs="?", default=".",
+                       help="project directory to check (default: current directory)")
+    check.set_defaults(func=cmd_check)
+
+    path = subparsers.add_parser("path", help="print the location of the packaged standard")
+    path.set_defaults(func=cmd_path)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if argv and argv[0] == "lint":
+        return run_lint(argv[1:])
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

md_ddl/docs/agents/CONVENTIONS.md

@@ -0,0 +1,88 @@
+# Agent Conventions
+
+Shared conventions for MD-DDL agents. Individual agent behaviour is defined in each agent's `AGENT.md`; this file covers cross-agent protocols that all work-performing agents follow.
+
+---
+
+## Handoff Artifact Files (V2)
+
+### Purpose
+
+When a user closes one agent session and opens a new session with a different agent, the inline handoff context block (V1) is lost with the conversation. Handoff artifact files provide **cross-session durability** — a persistent record of decisions, rejected alternatives, and task instructions that the receiving agent can read at startup.
+
+### Relationship to Inline Handoff Blocks (V1)
+
+V1 and V2 are complementary, not alternative:
+
+- **V1 (inline block)** — Required for all handoffs. Produced at the moment of handoff, lives in the conversation. Solves same-session transitions.
+- **V2 (file)** — Optional enhancement for cross-session durability. The sending agent writes the file when the user will switch to a new session. The receiving agent reads it at startup.
+
+The file format mirrors the inline block format — the difference is persistence. An agent implementing V2 produces the inline block first, then writes it to file if cross-session durability is needed.
+
+### File Location
+
+Handoff files live in the domain folder alongside `domain.md`:
+
+```
+examples/Financial Crime/
+  domain.md
+  entities/
+  handoff-to-artifact.md       ← written by Agent Ontology
+  handoff-to-governance.md     ← written by Agent Ontology or Agent Artifact
+  handoff-to-ontology.md       ← written by Agent Artifact, Architect, or Governance
+  handoff-to-architect.md      ← written by Agent Ontology
+```
+
+### File Naming
+
+`handoff-to-<agent-id>.md` — the file name encodes the destination. The receiving agent looks for its own handoff file on startup.
+
+### Frontmatter Schema
+
+Field | Type | Required | Description
+--- | --- | --- | ---
+`from` | string | Yes | Sending agent ID (e.g., `agent-ontology`)
+`to` | string | Yes | Receiving agent ID (e.g., `agent-artifact`)
+`domain` | string | Yes | Domain name
+`domain_path` | string | Yes | Relative path to domain.md
+`created` | date | Yes | Date the file was written
+`status` | enum | Yes | `pending` · `consumed` · `archived`
+
+### Content Block Template
+
+```markdown
+## Handoff Context — [Sending Agent] → [Receiving Agent]
+
+**Scope:** [entities, relationships, or aspects covered]
+
+**Key decisions:**
+- [decision and brief rationale — especially non-obvious choices]
+
+**Rejected alternatives:**
+- [what was considered but not chosen, and why]
+
+**Do not re-open:**
+- [questions already resolved that the next agent should accept as settled]
+
+**Task for next agent:**
+[Clear description of what needs to be done]
+```
+
+### Lifecycle
+
+1. **Sending agent writes** the file at handoff time, drawing on session context. Set `status: pending`.
+2. **Receiving agent reads** the file at session start — before loading domain files. Accept decisions marked "Do not re-open" as settled.
+3. **Receiving agent updates** `status: consumed` in the frontmatter after reading.
+4. **User or agent archives** the file (`status: archived`) when the work it describes is complete. Archived files are informational history.
+
+### Multiple Handoffs to the Same Agent
+
+Status-driven: only one `pending` file per destination agent is allowed. The previous handoff file must be `archived` before a new one is written. The receiving agent looks for the file with `status: pending`.
+
+### Cross-Domain Handoffs
+
+A consumer-aligned data product may span multiple domains. The handoff file lives in the primary domain's folder. The Scope section explicitly names any cross-domain entities referenced.
+
+### Version Control
+
+Handoff files should be committed to version control. They are living documentation of modelling decisions. `archived` files are the audit trail; `consumed` files show what the receiving agent was told. `.gitignore` should not exclude them.