overture-schema-codegen 0.1.1.dev0__tar.gz

overture_schema_codegen-0.1.1.dev0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: overture-schema-codegen
+Version: 0.1.1.dev0
+Summary: Code generator that produces documentation and code from Pydantic models
+License-Expression: MIT
+Requires-Dist: click>=8.1
+Requires-Dist: jinja2>=3.0
+Requires-Dist: overture-schema-cli>=0.1.1
+Requires-Dist: overture-schema-common>=0.1.1
+Requires-Dist: overture-schema-system>=0.1.1
+Requires-Dist: tomli>=2.0 ; python_full_version < '3.11'
+Requires-Dist: typing-extensions>=4.0
+Requires-Python: >=3.10

overture_schema_codegen-0.1.1.dev0/pyproject.toml

@@ -0,0 +1,62 @@
+[build-system]
+requires = ["uv_build>=0.11.32,<0.13"]
+build-backend = "uv_build"
+
+[project]
+dependencies = [
+    "click>=8.1",
+    "jinja2>=3.0",
+    "overture-schema-cli>=0.1.1",
+    "overture-schema-common>=0.1.1",
+    "overture-schema-system>=0.1.1",
+    "tomli>=2.0; python_version < '3.11'",
+    "typing-extensions>=4.0",
+]
+description = "Code generator that produces documentation and code from Pydantic models"
+version = "0.1.1.dev0"
+license = "MIT"
+name = "overture-schema-codegen"
+requires-python = ">=3.10"
+
+[project.scripts]
+overture-codegen = "overture.schema.codegen.cli:main"
+
+[tool.uv.sources.overture-schema-cli]
+workspace = true
+
+[tool.uv.sources.overture-schema-common]
+workspace = true
+
+[tool.uv.sources.overture-schema-system]
+workspace = true
+
+[tool.uv.sources.overture-schema-theme-addresses]
+workspace = true
+
+[tool.uv.sources.overture-schema-theme-base]
+workspace = true
+
+[tool.uv.sources.overture-schema-theme-buildings]
+workspace = true
+
+[tool.uv.sources.overture-schema-theme-divisions]
+workspace = true
+
+[tool.uv.sources.overture-schema-theme-places]
+workspace = true
+
+[tool.uv.sources.overture-schema-theme-transportation]
+workspace = true
+
+[tool.uv.build-backend]
+module-name = "overture.schema.codegen"
+
+[dependency-groups]
+test = [
+    "overture-schema-theme-addresses",
+    "overture-schema-theme-base",
+    "overture-schema-theme-buildings",
+    "overture-schema-theme-divisions",
+    "overture-schema-theme-places",
+    "overture-schema-theme-transportation",
+]

overture_schema_codegen-0.1.1.dev0/pyproject.toml.orig

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["uv_build>=0.11.32,<0.13"]
+build-backend = "uv_build"
+
+[project]
+dependencies = [
+    "click>=8.1",
+    "jinja2>=3.0",
+    "overture-schema-cli>=0.1.1",
+    "overture-schema-common>=0.1.1",
+    "overture-schema-system>=0.1.1",
+    "tomli>=2.0; python_version < '3.11'",
+    "typing-extensions>=4.0",
+]
+description = "Code generator that produces documentation and code from Pydantic models"
+version = "0.1.1.dev0"
+license = "MIT"
+name = "overture-schema-codegen"
+requires-python = ">=3.10"
+
+[project.scripts]
+overture-codegen = "overture.schema.codegen.cli:main"
+
+[tool.uv.sources]
+overture-schema-cli = { workspace = true }
+overture-schema-common = { workspace = true }
+overture-schema-system = { workspace = true }
+overture-schema-theme-addresses = { workspace = true }
+overture-schema-theme-base = { workspace = true }
+overture-schema-theme-buildings = { workspace = true }
+overture-schema-theme-divisions = { workspace = true }
+overture-schema-theme-places = { workspace = true }
+overture-schema-theme-transportation = { workspace = true }
+
+[dependency-groups]
+test = [
+    "overture-schema-theme-addresses",
+    "overture-schema-theme-base",
+    "overture-schema-theme-buildings",
+    "overture-schema-theme-divisions",
+    "overture-schema-theme-places",
+    "overture-schema-theme-transportation",
+]
+
+[tool.uv.build-backend]
+module-name = "overture.schema.codegen"

overture_schema_codegen-0.1.1.dev0/src/overture/schema/codegen/__init__.py

@@ -0,0 +1 @@
+"""Code generator for Overture Schema Pydantic models."""

overture_schema_codegen-0.1.1.dev0/src/overture/schema/codegen/cli.py

@@ -0,0 +1,228 @@
+"""CLI entrypoint for schema code generation."""
+
+import json
+import logging
+from collections.abc import Mapping
+from pathlib import Path, PurePosixPath
+
+import click
+
+from overture.schema.cli.tag_options import build_selector, tag_selection_options
+from overture.schema.system.discovery import (
+    discover_models,
+    filter_models,
+    split_entry_point,
+)
+
+from .extraction.specs import ModelSpec, SupplementarySpec, TypeIdentity
+from .layout.module_layout import (
+    OUTPUT_ROOT,
+    compute_schema_root,
+    entry_point_module,
+)
+from .markdown.pipeline import generate_markdown_pages
+from .pyspark.pipeline import generate_pyspark_modules
+from .spec_discovery import extract_alias_spec, extract_model_spec
+
+log = logging.getLogger(__name__)
+
+__all__ = ["cli"]
+
+_OUTPUT_FORMATS = ("markdown", "pyspark")
+
+_FEATURE_FRONTMATTER = "---\nsidebar_position: 1\n---\n\n"
+
+
+def _write_output(
+    content: str,
+    output_dir: Path | None,
+    output_path: PurePosixPath,
+) -> None:
+    """Write content to a file under output_dir, or stdout."""
+    if output_dir:
+        file_path = output_dir / output_path
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_text(content)
+    else:
+        click.echo(content)
+        click.echo()  # separate entries with a blank line in stdout mode
+
+
+@click.group()
+def cli() -> None:
+    """Overture Schema code generator.
+
+    Generate documentation and code from Pydantic schema models.
+    """
+
+
+@cli.command("list")
+def list_models() -> None:
+    """List all discovered models."""
+    models = discover_models()
+    # Name every entry from its entry point, not the loaded object: a
+    # discriminated union loads as an `Annotated[...]` alias with no
+    # `__name__`, so `str(model)` would print the whole type expression.
+    names = []
+    for key in models:
+        _, class_name = split_entry_point(key.entry_point)
+        names.append(class_name)
+    for name in sorted(names):
+        click.echo(name)
+
+
+@cli.command()
+@click.option(
+    "--format",
+    "output_format",
+    required=True,
+    type=click.Choice(_OUTPUT_FORMATS),
+    help="Output format",
+)
+@tag_selection_options
+@click.option(
+    "--output-dir",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Write output files directly into this directory (default: stdout). "
+    "For pyspark, writes expression modules (*.py). "
+    "For markdown, writes theme subdirectories.",
+)
+@click.option(
+    "--test-output-dir",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Write test modules (test_*.py) into this directory (pyspark only).",
+)
+def generate(
+    output_format: str,
+    tags: tuple[str, ...],
+    filters: tuple[str, ...],
+    excludes: tuple[str, ...],
+    output_dir: Path | None,
+    test_output_dir: Path | None,
+) -> None:
+    """Generate code/docs from discovered models."""
+    if output_format != "pyspark" and test_output_dir is not None:
+        raise click.UsageError("--test-output-dir is only valid with --format pyspark")
+
+    all_models = discover_models()
+
+    models = filter_models(all_models, build_selector(tags, filters, excludes))
+
+    if output_dir:
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+    model_specs: list[ModelSpec] = [
+        spec
+        for key, entry in models.items()
+        if (spec := extract_model_spec(key, entry)) is not None
+    ]
+
+    if output_format == "pyspark":
+        _generate_pyspark(model_specs, output_dir, test_output_dir)
+    else:
+        # RootModel entry points yield no ModelSpec, so they document as
+        # named aliases -- reachable no other way, since a RootModel field
+        # unwraps to its bare shape and names no type.
+        external_specs: dict[TypeIdentity, SupplementarySpec] = {
+            alias.identity: alias
+            for entry in models.values()
+            if (alias := extract_alias_spec(entry)) is not None
+        }
+        module_paths = [entry_point_module(k.entry_point) for k in all_models]
+        schema_root = compute_schema_root(module_paths)
+        _generate_markdown(model_specs, schema_root, output_dir, external_specs)
+
+
+def _generate_markdown(
+    model_specs: list[ModelSpec],
+    schema_root: str,
+    output_dir: Path | None,
+    external_specs: Mapping[TypeIdentity, SupplementarySpec],
+) -> None:
+    """Generate markdown with directory layout and placement-aware links."""
+    pages = generate_markdown_pages(
+        model_specs, schema_root, external_specs=external_specs
+    )
+
+    for page in pages:
+        content = (
+            f"{_FEATURE_FRONTMATTER}{page.content}" if page.is_model else page.content
+        )
+        _write_output(content, output_dir, page.path)
+
+    if output_dir:
+        feature_paths = {page.path for page in pages if page.is_model}
+        all_paths = {page.path for page in pages}
+        _write_category_files(output_dir, all_paths, feature_paths)
+
+
+def _generate_pyspark(
+    model_specs: list[ModelSpec],
+    output_dir: Path | None,
+    test_output_dir: Path | None = None,
+) -> None:
+    """Generate PySpark validation modules.
+
+    Output is syntactically valid Python; we assume a code formatter runs
+    over the written directories afterwards to match existing conventions.
+    """
+    modules = generate_pyspark_modules(model_specs)
+    for mod in modules.source:
+        _write_output(mod.content, output_dir, mod.path)
+    if test_output_dir is not None:
+        for mod in modules.test:
+            _write_output(mod.content, test_output_dir, mod.path)
+
+
+def _ancestor_dirs(paths: set[PurePosixPath]) -> set[PurePosixPath]:
+    """Collect all ancestor directories for a set of file paths."""
+    dirs: set[PurePosixPath] = set()
+    for path in paths:
+        parent = path.parent
+        while parent != OUTPUT_ROOT:
+            dirs.add(parent)
+            parent = parent.parent
+    return dirs
+
+
+def _top_level_positions(
+    dirs: set[PurePosixPath],
+    feature_paths: set[PurePosixPath],
+) -> dict[PurePosixPath, int]:
+    """Assign sidebar positions: feature dirs first, then non-feature, both alphabetical."""
+    feature_dir_names = {p.parts[0] for p in feature_paths}
+    top_level = sorted(d for d in dirs if d.parent == OUTPUT_ROOT)
+    feature_dirs = [d for d in top_level if d.name in feature_dir_names]
+    non_feature_dirs = [d for d in top_level if d.name not in feature_dir_names]
+    return {d: i for i, d in enumerate(feature_dirs + non_feature_dirs, start=1)}
+
+
+def _write_category_files(
+    output_dir: Path,
+    all_paths: set[PurePosixPath],
+    feature_paths: set[PurePosixPath],
+) -> None:
+    """Write _category_.json files for Docusaurus sidebar navigation."""
+    dirs = _ancestor_dirs(all_paths)
+    positions = _top_level_positions(dirs, feature_paths)
+
+    for dir_path in sorted(dirs):
+        label = dir_path.name.replace("_", " ").title()
+        category: dict[str, object] = {"label": label}
+        if dir_path in positions:
+            category["position"] = positions[dir_path]
+
+        file_path = output_dir / dir_path / "_category_.json"
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_text(json.dumps(category, indent=2) + "\n")
+
+
+def main() -> None:
+    """Run the CLI entry point."""
+    cli()
+
+
+if __name__ == "__main__":
+    main()

overture_schema_codegen-0.1.1.dev0/src/overture/schema/codegen/extraction/docstring.py

@@ -0,0 +1,46 @@
+"""Docstring extraction and cleaning utilities."""
+
+import inspect
+from enum import Enum
+from typing import NewType
+
+__all__ = ["clean_docstring", "first_docstring_line", "is_custom_docstring"]
+
+
+# Probe auto-generated docstrings so we can distinguish them from explicit ones.
+# Both Enum and NewType generate default docstrings that vary by Python version;
+# capturing at import time adapts automatically if the format changes.
+class _DocstringProbeEnum(Enum):
+    pass
+
+
+_ENUM_DEFAULT_DOCSTRING = _DocstringProbeEnum.__doc__
+del _DocstringProbeEnum
+_NewtypeProbe = NewType("_NewtypeProbe", int)
+_NEWTYPE_DEFAULT_DOCSTRING = _NewtypeProbe.__doc__
+del _NewtypeProbe
+
+
+def clean_docstring(doc: str | None) -> str | None:
+    """Return cleaned docstring, or None if absent or whitespace-only."""
+    if not doc:
+        return None
+    cleaned = inspect.cleandoc(doc)
+    return cleaned or None
+
+
+def first_docstring_line(doc: str | None) -> str | None:
+    """Return the first line of a docstring, or None if absent."""
+    cleaned = clean_docstring(doc)
+    if not cleaned:
+        return None
+    return cleaned.split("\n")[0]
+
+
+def is_custom_docstring(doc: str | None, inherited_doc: str | None = None) -> bool:
+    """Check if a docstring was explicitly written, not auto-generated or inherited."""
+    return bool(doc) and doc not in (
+        _ENUM_DEFAULT_DOCSTRING,
+        _NEWTYPE_DEFAULT_DOCSTRING,
+        inherited_doc,
+    )

overture_schema_codegen-0.1.1.dev0/src/overture/schema/codegen/extraction/enum_extraction.py

@@ -0,0 +1,40 @@
+"""Enum extraction."""
+
+from enum import Enum
+
+from .docstring import clean_docstring, is_custom_docstring
+from .specs import EnumMemberSpec, EnumSpec
+
+__all__ = ["extract_enum"]
+
+
+def extract_enum(enum_class: type[Enum]) -> EnumSpec:
+    """Extract enum specification from an Enum class.
+
+    Handles both simple str Enums and DocumentedEnums where members
+    have per-value descriptions via the __doc__ attribute.
+    """
+    class_doc = enum_class.__doc__
+    description = clean_docstring(class_doc) if is_custom_docstring(class_doc) else None
+
+    members: list[EnumMemberSpec] = []
+    for member in enum_class:
+        member_doc = getattr(member, "__doc__", None)
+        member_description = (
+            member_doc if is_custom_docstring(member_doc, class_doc) else None
+        )
+
+        members.append(
+            EnumMemberSpec(
+                name=member.name,
+                value=str(member.value),
+                description=member_description,
+            )
+        )
+
+    return EnumSpec(
+        name=enum_class.__name__,
+        description=description,
+        members=members,
+        source_type=enum_class,
+    )