msra-codegen 0.1.0__py3-none-any.whl

msra_codegen/package_metadata.py

@@ -0,0 +1,306 @@
+from __future__ import annotations
+
+import json
+import re
+from datetime import datetime
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+from urllib.request import urlopen
+
+from packaging.version import Version
+from jinja2 import TemplateNotFound
+
+from .core_naming import root_client_class_name
+from .file_utils import write_text
+from .generator_config import config_section
+from .template_engine import render_template
+
+
+def render_pyproject(project: dict[str, Any], package_name: str) -> str:
+    client_class_name = root_client_class_name(project)
+    authors = project["app"].get("authors", [])
+    min_required_python = str(project["app"].get("min_required_python", "3.10") or "3.10").strip()
+    description = str(project["app"].get("description", "") or "").strip()
+    runtime_dependencies = collect_runtime_dependencies(project)
+    licenses_config = config_section("licenses")
+    return render_template(
+        "pyproject.toml.tpl",
+        {
+            "authors_block": render_authors_block(authors),
+            "description": json.dumps(description, ensure_ascii=False),
+            "license": project["app"].get("license", licenses_config.get("default", "MIT")),
+            "keywords_block": render_keywords_block(project["app"].get("keywords", [])),
+            "classifiers_block": render_classifiers_block(min_required_python),
+            "requires_python": f">={min_required_python}",
+            "dependencies_block": render_toml_string_list("dependencies", runtime_dependencies),
+            "package_name": package_name,
+            "autotest_start_class": f"{package_name}.{client_class_name}",
+            "mypy_block": render_mypy_block(),
+            "ruff_block": render_ruff_block(),
+        },
+    )
+
+
+def render_authors_block(authors: Any) -> str:
+    items: list[str] = []
+    if isinstance(authors, list):
+        for author in authors:
+            if not isinstance(author, dict):
+                continue
+            fields = [f'name = {json.dumps(str(author.get("name", "")))}']
+            email = str(author.get("email", "")).strip()
+            if email:
+                fields.append(f'email = {json.dumps(email)}')
+            items.append("    { " + ", ".join(fields) + " }")
+    if not items:
+        return "authors = []"
+    return "authors = [\n" + ",\n".join(items) + "\n]"
+
+
+def render_keywords_block(keywords: Any) -> str:
+    return render_toml_string_list("keywords", keywords)
+
+
+def render_ruff_block() -> str:
+    ruff_config = config_section("ruff")
+    line_length = ruff_config.get("line_length")
+    if not isinstance(line_length, int):
+        raise RuntimeError("ruff.line_length must be an integer.")
+
+    lint_config = ruff_config.get("lint")
+    if not isinstance(lint_config, dict):
+        raise RuntimeError("ruff.lint must be a table.")
+
+    select = lint_config.get("select")
+    if not isinstance(select, list) or not select:
+        raise RuntimeError("ruff.lint.select must be a non-empty list.")
+
+    ignore = lint_config.get("ignore", [])
+    if not isinstance(ignore, list):
+        raise RuntimeError("ruff.lint.ignore must be a list.")
+
+    return "\n".join(
+        [
+            "[tool.ruff]",
+            f"line-length = {line_length}",
+            "",
+            "[tool.ruff.lint]",
+            render_toml_string_list("select", select),
+            render_toml_string_list("ignore", ignore),
+        ]
+    )
+
+
+def render_mypy_block() -> str:
+    mypy_config = config_section("mypy")
+    if not isinstance(mypy_config, dict):
+        raise RuntimeError("mypy must be a table.")
+
+    items: list[str] = []
+    for key, value in mypy_config.items():
+        if isinstance(value, bool):
+            rendered_value = "true" if value else "false"
+        elif isinstance(value, (int, float)) and not isinstance(value, bool):
+            rendered_value = str(value)
+        elif isinstance(value, str):
+            rendered_value = json.dumps(value)
+        elif isinstance(value, list):
+            rendered_items: list[str] = []
+            for item in value:
+                if isinstance(item, bool):
+                    rendered_items.append("true" if item else "false")
+                elif isinstance(item, (int, float)) and not isinstance(item, bool):
+                    rendered_items.append(str(item))
+                elif isinstance(item, str):
+                    rendered_items.append(json.dumps(item))
+                else:
+                    raise RuntimeError(f"mypy.{key} contains an unsupported value type.")
+            rendered_value = "["
+            if rendered_items:
+                rendered_value += ", ".join(rendered_items)
+            rendered_value += "]"
+        else:
+            raise RuntimeError(f"mypy.{key} contains an unsupported value type.")
+        items.append(f"{key} = {rendered_value}")
+
+    if not items:
+        raise RuntimeError("mypy must not be empty.")
+
+    return "\n".join(["[tool.mypy]", *items])
+
+
+@lru_cache(maxsize=1)
+def latest_supported_python_minor() -> int:
+    families = load_python_release_families()
+    if len(families) < 2:
+        raise RuntimeError(
+            "Could not determine the penultimate Python 3 minor family from the python.org releases API."
+        )
+    return families[1][1]
+
+
+def load_python_release_families() -> list[tuple[int, int]]:
+    python_config = config_section("python")
+    try:
+        with urlopen(
+            str(python_config.get("releases_api_url", "https://www.python.org/api/v2/downloads/release/")),
+            timeout=float(python_config.get("releases_api_timeout_seconds", 15.0)),
+        ) as response:
+            data = json.load(response)
+    except OSError as exc:  # pragma: no cover - depends on network availability
+        raise RuntimeError(
+            "Failed to load python.org releases API."
+        ) from exc
+    if not isinstance(data, list):
+        raise RuntimeError("Unexpected response format from python.org releases API.")
+
+    families = {
+        (version.major, version.minor)
+        for version in extract_python_release_versions(data)
+    }
+    return sorted(families, reverse=True)
+
+
+def extract_python_release_versions(data: Any) -> list[Version]:
+    versions: list[Version] = []
+    for item in data:
+        if not isinstance(item, dict):
+            continue
+        name = str(item.get("name", ""))
+        if not name.startswith("Python "):
+            continue
+        raw_version = name.removeprefix("Python ").strip()
+        try:
+            version = Version(raw_version)
+        except Exception:
+            continue
+        if version.major != 3:
+            continue
+        if version.is_prerelease or version.is_devrelease:
+            continue
+        versions.append(version)
+    return versions
+
+
+def render_classifiers_block(min_required_python: str) -> str:
+    classifiers_config = config_section("classifiers")
+    match = re.fullmatch(r"(\d+)\.(\d+)", min_required_python.strip())
+    if not match:
+        version_labels = [f"Programming Language :: Python :: {min_required_python.strip()}"]
+    else:
+        major = int(match.group(1))
+        start_minor = int(match.group(2))
+        version_labels = [f"Programming Language :: Python :: {major}"]
+        if major == 3:
+            end_minor = max(start_minor, latest_supported_python_minor())
+            version_labels.extend(
+                f"Programming Language :: Python :: 3.{minor}"
+                for minor in range(start_minor, end_minor + 1)
+            )
+        else:
+            version_labels.append(f"Programming Language :: Python :: {major}.{start_minor}")
+    version_labels.extend(
+        [
+            *[str(item) for item in classifiers_config.get("static", []) if str(item).strip()],
+        ]
+    )
+    return render_toml_string_list("classifiers", version_labels)
+
+
+def render_toml_string_list(key: str, values: Any) -> str:
+    items: list[str] = []
+    if isinstance(values, list):
+        for value in values:
+            text = str(value).strip()
+            if not text:
+                continue
+            items.append(json.dumps(text))
+    if not items:
+        return f"{key} = []"
+    if len(items) == 1:
+        return f"{key} = [{items[0]}]"
+    return f"{key} = [\n    " + ",\n    ".join(items) + "\n]"
+
+
+def collect_runtime_dependencies(project: dict[str, Any]) -> list[str]:
+    runtime_config = config_section("runtime_dependencies")
+    dependencies = [str(item).strip() for item in runtime_config.get("base", []) if str(item).strip()]
+    if any(func.get("transport") == "direct" for func in project.get("functions", [])):
+        dependencies.extend(
+            str(item).strip() for item in runtime_config.get("direct", []) if str(item).strip()
+        )
+    if project.get("app", {}).get("abstractions"):
+        dependencies.append("pydantic")
+    return dependencies
+
+
+def collect_dev_dependencies() -> list[str]:
+    dev_config = config_section("development_dependencies")
+    return [str(item).strip() for item in dev_config.get("base", []) if str(item).strip()]
+
+
+def render_requirements_txt(project: dict[str, Any]) -> str:
+    return "\n".join(collect_runtime_dependencies(project)) + "\n"
+
+
+def render_requirements_dev_txt(project: dict[str, Any]) -> str:
+    lines = [
+        "-r requirements.txt",
+        "-r docs/requirements.txt",
+        *collect_dev_dependencies(),
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def write_root_license(output_dir: Path, project: dict[str, Any]) -> None:
+    licenses_config = config_section("licenses")
+    license_name = resolve_license_template_name(
+        str(project["app"].get("license", licenses_config.get("default", "MIT")) or "").strip()
+        or str(licenses_config.get("default", "MIT"))
+    )
+    authors = project["app"].get("authors", [])
+    license_text = render_license_text(license_name, authors)
+    write_text(output_dir / "LICENSE", license_text)
+
+
+def resolve_license_template_name(license_name: str) -> str:
+    normalized = license_name.strip()
+    aliases = config_section("licenses").get("aliases", {})
+    if isinstance(aliases, dict) and normalized in aliases:
+        return str(aliases[normalized]).strip()
+    return normalized
+
+
+def render_license_text(license_name: str, authors: Any) -> str:
+    context = {}
+    if license_name == "MIT":
+        context = {
+            "copyright_holders": format_copyright_holders(authors),
+            "year": datetime.now().year,
+        }
+    template_name = f"licenses/{license_name}.txt.tpl"
+    try:
+        return render_template(template_name, context)
+    except TemplateNotFound as exc:  # pragma: no cover - guardrail for unsupported licenses
+        raise RuntimeError(
+            f'Missing local license template "{template_name}". Add it under msra_codegen/templates/licenses/.'
+        ) from exc
+
+
+def format_copyright_holders(authors: Any) -> str:
+    names: list[str] = []
+    if isinstance(authors, list):
+        for author in authors:
+            if not isinstance(author, dict):
+                continue
+            name = str(author.get("name", "")).strip()
+            if name:
+                names.append(name)
+    if not names:
+        return "The authors"
+    if len(names) == 1:
+        return names[0]
+    if len(names) == 2:
+        return " and ".join(names)
+    return ", ".join(names[:-1]) + ", and " + names[-1]

msra_codegen/package_writer.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import Any
+
+from .codegen_context import (
+    build_abstraction_package_context,
+    collect_abstraction_scripts,
+    collect_extractor_scripts,
+    collect_goto_pipeline_scripts,
+    collect_warmup_scripts,
+    render_endpoints_init,
+    render_init,
+    render_manager_template,
+    write_group_package,
+)
+from .gitignore import generate_gitignore_project
+from .github_workflows import generate_github_workflows_project
+from .issue_templates import generate_github_issue_templates_project
+from .core_naming import abstraction_module_name_from_path, normalize_abstraction_path, normalize_script_path
+from .file_utils import write_text
+from .python_formatting import format_python_tree
+from .package_metadata import render_pyproject, render_requirements_dev_txt, render_requirements_txt, write_root_license
+from .project_model import build_group_tree, top_level_groups
+from .tests_generator import build_tests_project_context, generate_tests_project
+from .template_engine import render_template
+
+
+def ensure_package_inits(target_dir: Path, package_root: Path) -> None:
+    current = target_dir
+    while current != package_root and package_root in current.parents:
+        init_file = current / "__init__.py"
+        if not init_file.exists():
+            write_text(init_file, "")
+        current = current.parent
+
+
+def generate_project(
+    project: dict[str, Any],
+    output_dir: Path,
+    source_root: Path | None = None,
+) -> None:
+    output_dir = output_dir.resolve()
+    source_root = source_root.resolve() if source_root is not None else Path(project["source_path"]).resolve().parent
+    package_name = str(project["app"].get("package_name") or "").strip()
+    if not package_name:
+        raise ValueError('app.package_name is required and must be set explicitly in the source MSRA file.')
+    group_tree = build_group_tree(project)
+
+    package_root = output_dir / package_name
+    abstraction_root = package_root / "abstraction"
+    endpoints_root = package_root / "endpoints"
+    pipelines_root = package_root / "pipelines"
+    legacy_postprocess_root = package_root / "postprocess"
+    extractors_root = package_root / "extractors"
+    package_root.mkdir(parents=True, exist_ok=True)
+    if abstraction_root.exists():
+        shutil.rmtree(abstraction_root)
+    abstraction_root.mkdir(parents=True, exist_ok=True)
+    if pipelines_root.exists():
+        shutil.rmtree(pipelines_root)
+    pipelines_root.mkdir(parents=True, exist_ok=True)
+    if legacy_postprocess_root.exists():
+        shutil.rmtree(legacy_postprocess_root)
+    if extractors_root.exists():
+        shutil.rmtree(extractors_root)
+    extractors_root.mkdir(parents=True, exist_ok=True)
+    for stale_pipeline_file in [package_root / "warmup.py", package_root / "goto_pipeline.py"]:
+        if stale_pipeline_file.exists():
+            stale_pipeline_file.unlink()
+
+    tests_context = build_tests_project_context(project, package_name)
+
+    write_text(
+        output_dir / "pyproject.toml",
+        render_pyproject(project, package_name),
+    )
+    write_text(
+        output_dir / "requirements.txt",
+        render_requirements_txt(project),
+    )
+    write_text(
+        output_dir / "requirements-dev.txt",
+        render_requirements_dev_txt(project),
+    )
+    write_text(package_root / "__init__.py", render_init(project, package_name))
+    stale_abstraction_file = package_root / "abstraction.py"
+    if stale_abstraction_file.exists():
+        stale_abstraction_file.unlink()
+    abstraction_context = build_abstraction_package_context(project)
+    copied_abstraction_modules: set[str] = set()
+    for abstraction_script in collect_abstraction_scripts(project):
+        normalized_abstraction = normalize_abstraction_path(abstraction_script)
+        source = Path(normalized_abstraction)
+        if not source.is_absolute():
+            source = source_root / source
+        module_name = abstraction_module_name_from_path(normalized_abstraction)
+        if module_name in {"__init__", "regexes"}:
+            raise ValueError(
+                f'Abstraction file "{abstraction_script}" resolves to reserved module name "{module_name}".'
+            )
+        if module_name in copied_abstraction_modules:
+            raise ValueError(f'Duplicate abstraction module name "{module_name}" derived from "{abstraction_script}".')
+        copied_abstraction_modules.add(module_name)
+        target = abstraction_root / f"{module_name}.py"
+        shutil.copyfile(source, target)
+    write_text(
+        abstraction_root / "__init__.py",
+        render_template("abstraction/__init__.py.tpl", abstraction_context),
+    )
+    write_text(
+        abstraction_root / "regexes.py",
+        render_template("abstraction/regexes.py.tpl", abstraction_context),
+    )
+    write_text(
+        package_root / "manager.py",
+        render_manager_template(
+            project,
+            package_name,
+            group_tree,
+            autotest_function_ids=tests_context["autotest_function_ids"],
+        ),
+    )
+    if endpoints_root.exists():
+        shutil.rmtree(endpoints_root)
+    endpoints_root.mkdir(parents=True, exist_ok=True)
+    write_text(endpoints_root / "__init__.py", render_endpoints_init(project, package_name, group_tree))
+    for group_node in top_level_groups(group_tree):
+        write_group_package(
+            group_node,
+            project,
+            package_name,
+            endpoints_root,
+            autotest_function_ids=tests_context["autotest_function_ids"],
+        )
+
+    for script in dict.fromkeys(collect_warmup_scripts(project) + collect_goto_pipeline_scripts(project)):
+        source = source_root / script
+        target = pipelines_root / normalize_script_path(script)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        ensure_package_inits(target.parent, package_root)
+        shutil.copyfile(source, target)
+
+    for script in dict.fromkeys(collect_extractor_scripts(project)):
+        source = source_root / script
+        target = package_root / normalize_script_path(script)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(source, target)
+
+    legacy_license_dir = output_dir / "LICENSES"
+    if legacy_license_dir.exists():
+        shutil.rmtree(legacy_license_dir)
+    write_root_license(output_dir, project)
+    generate_gitignore_project(output_dir)
+    generate_github_workflows_project(project, output_dir, package_name)
+    generate_github_issue_templates_project(project, output_dir)
+
+    from .docs_generator import generate_docs_project
+
+    generate_docs_project(
+        project,
+        output_dir,
+        package_name,
+        group_tree,
+        tests_context=tests_context,
+    )
+    generate_tests_project(
+        project,
+        output_dir,
+        package_name,
+        group_tree,
+        tests_context=tests_context,
+    )
+    format_python_tree(output_dir)