msra-codegen 0.1.0__py3-none-any.whl

msra_codegen/config.toml

@@ -0,0 +1,164 @@
+[python]
+releases_api_url = "https://www.python.org/api/v2/downloads/release/"
+releases_api_timeout_seconds = 15.0
+
+[runtime_dependencies]
+base = [
+    "camoufox[geoip]",
+    "human_requests",
+    "Pillow",
+    "rich",
+]
+direct = [
+    "aiohttp",
+    "aiohttp-retry",
+]
+
+[development_dependencies]
+base = [
+    "pytest",
+    "pytest-anyio",
+    "pytest-jsonschema-snapshot",
+    "ruff",
+    "mypy",
+]
+
+[gitignore]
+patterns = [
+    "# Python bytecode",
+    "__pycache__/",
+    "*.py[cod]",
+    "*.pyd",
+    "# Virtual environments",
+    ".venv/",
+    "venv/",
+    "env/",
+    "# Test, lint, and coverage caches",
+    ".pytest_cache/",
+    ".mypy_cache/",
+    ".ruff_cache/",
+    ".coverage",
+    "coverage.xml",
+    "coverage.svg",
+    "htmlcov/",
+    "# Build artifacts",
+    "build/",
+    "dist/",
+    "*.egg-info/",
+    "# Generated project noise",
+    "merged.msra",
+    "error.log",
+    "screenshot.png",
+    "docs/_build/",
+]
+
+[classifiers]
+static = [
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX :: Linux",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Internet",
+    "Topic :: Utilities",
+]
+
+[licenses]
+default = "MIT"
+aliases = { "GPL-3.0" = "GPL-3.0-or-later", "GPL-3.0+" = "GPL-3.0-or-later" }
+
+[readme]
+principle_text = "Библиотека полностью повторяет сетевую работу обычного пользователя на сайте."
+
+[[readme.badges]]
+label = "Ruff"
+badge_url = "https://img.shields.io/badge/linting-Ruff-blue?logo=ruff&logoColor=white"
+url = "https://github.com/astral-sh/ruff"
+
+[[readme.socials]]
+key = "discord"
+label = "Discord"
+badge_url = "https://img.shields.io/discord/792572437292253224?label=Discord&labelColor=%232c2f33&color=%237289da"
+
+[[readme.socials]]
+key = "telegram"
+label = "Telegram"
+badge_url = "https://img.shields.io/badge/Telegram-24A1DE"
+
+[github.workflows.tests]
+name = "tests"
+schedule_cron = "17 3 * * *"
+runner = ["self-hosted"]
+python_version = "3.12"
+checkout_action = "actions/checkout@v4"
+setup_python_action = "actions/setup-python@v5"
+install_commands = [
+    "python -m venv venv",
+    "venv/bin/python -m pip install -U pip",
+    "venv/bin/python -m pip install -r requirements-dev.txt",
+]
+run_command_shell = "bash -e -lc"
+headed_run_command_shell = "xvfb-run -a bash -e -lc"
+run_commands = [
+    "set -o pipefail",
+    "venv/bin/python -m pytest --jsss-ci-cd --save-original --tb=short 2>&1 | tee error.log",
+]
+report_playwright_failure_action = "Miskler/human-requests-bot@v11"
+report_schema_action = "Miskler/pytest-jsonschema-snapshot-bot@v14"
+github_token_secret = "GITHUB_TOKEN"
+log_path = "error.log"
+screenshot_path = "screenshot.png"
+
+[github.workflows.publish]
+name = "publish"
+python_version = "3.12"
+branch = "main"
+checkout_action = "actions/checkout@v4"
+setup_python_action = "actions/setup-python@v5"
+setup_python_cache = "pip"
+upload_pages_action = "actions/upload-pages-artifact@v3"
+deploy_pages_action = "actions/deploy-pages@v4"
+pypi_action = "pypa/gh-action-pypi-publish@release/v1"
+target_options = ["docs", "package", "all"]
+pages_environment_name = "github-pages"
+pypi_environment_name = "pypi"
+github_token_secret = "GITHUB_TOKEN"
+
+[github.workflows.source_sync]
+name = "source-sync"
+source_branch = "source"
+target_branch = "main"
+logic_repository = "Open-Inflation/engine-reverse-ide"
+logic_ref = "main"
+checkout_action = "actions/checkout@v4"
+setup_python_action = "actions/setup-python@v5"
+python_version = "3.12"
+generator_requirements_path = "msra_codegen/requirements.txt"
+repo_token_secret = "SOURCE_SYNC_TOKEN"
+commit_user_name = "msra-sync-bot"
+commit_user_email = "msra-sync-bot@users.noreply.github.com"
+
+[[validation.checks]]
+name = "python-syntax"
+argv = ["{python_executable}", "-m", "compileall", "-q"]
+targets = ["{package_name}", "tests", "example.py", "docs/source/conf.py"]
+
+[ruff]
+line_length = 200
+
+[ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+ignore = ["RUF001", "RUF002", "RUF003"]
+
+[mypy]
+ignore_missing_imports = true
+
+[[validation.checks]]
+name = "ruff"
+argv = ["{python_executable}", "-m", "ruff", "check"]
+targets = ["{package_name}", "tests", "example.py", "docs/source/conf.py"]
+
+[[validation.checks]]
+name = "mypy"
+argv = ["{python_executable}", "-m", "mypy"]
+targets = ["{package_name}"]

msra_codegen/core_naming.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from .python_render import get_plain_value
+
+
+def snake_case(text: str) -> str:
+    text = re.sub(r"(?<!^)(?=[A-Z])", "_", text).replace("-", "_")
+    text = re.sub(r"[^A-Za-z0-9_]+", "_", text)
+    return text.strip("_").lower() or "generated"
+
+
+def pascal_case(text: str) -> str:
+    parts = re.split(r"[^A-Za-z0-9]+|_", text)
+    cleaned = []
+    for part in parts:
+        if not part:
+            continue
+        cleaned.append(part[:1].upper() + part[1:].lower())
+    return "".join(cleaned) or "Generated"
+
+
+def root_client_class_name(project: dict[str, Any]) -> str:
+    name = str(project["app"]["name"])
+    if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
+        return name
+    return pascal_case(name)
+
+
+def normalize_script_path(script_path: str) -> str:
+    normalized = script_path.strip().replace("\\", "/")
+    while normalized.startswith("./"):
+        normalized = normalized[2:]
+    normalized = normalized.strip("/")
+    if normalized.startswith("pipelines/"):
+        normalized = normalized[len("pipelines/"):]
+    return normalized
+
+
+def normalize_abstraction_path(abstraction_path: str) -> str:
+    normalized = abstraction_path.strip().replace("\\", "/")
+    while normalized.startswith("./"):
+        normalized = normalized[2:]
+    return normalized.strip("/")
+
+
+def abstraction_module_name_from_path(abstraction_path: str) -> str:
+    normalized = normalize_abstraction_path(abstraction_path)
+    if normalized.endswith(".py"):
+        normalized = normalized[:-3]
+    normalized = normalized.strip("/")
+    if not normalized:
+        return "generated"
+    return Path(normalized).name
+
+
+def module_file_name_for_group(path: list[str]) -> str:
+    if not path:
+        return "generated.py"
+    return f"{snake_case(path[-1])}.py"
+
+
+def package_dir_for_group(path: list[str]) -> Path:
+    return Path(*[snake_case(segment) for segment in path]) if path else Path()
+
+
+def module_import_name_for_group(group_node: dict[str, Any]) -> str:
+    path = group_node.get("path", [])
+    if not path:
+        return "generated"
+    if group_node.get("children"):
+        return snake_case(path[-1])
+    return module_file_name_for_group(path)[:-3]
+
+
+def module_package_depth_for_group(group_node: dict[str, Any]) -> int:
+    path = group_node.get("path", [])
+    return len(path) + (1 if group_node.get("children") else 0)
+
+
+def module_output_dir_for_group(group_node: dict[str, Any], endpoints_root: Path) -> Path:
+    path = group_node.get("path", [])
+    if group_node.get("children"):
+        return endpoints_root / package_dir_for_group(path)
+    if path:
+        return endpoints_root / package_dir_for_group(path[:-1])
+    return endpoints_root
+
+
+def base_class_name_for_group(path: list[str]) -> str:
+    if not path:
+        return "GeneratedGroup"
+    return pascal_case(path[-1])
+
+
+def class_name_for_group(path: list[str]) -> str:
+    return f"Class{base_class_name_for_group(path)}"
+
+
+def field_name_for_group(path: list[str]) -> str:
+    if not path:
+        return "Group"
+    return str(path[-1])
+
+
+def group_public_import_path(package_name: str, group_path: list[str]) -> str:
+    segments = [snake_case(segment) for segment in group_path]
+    if not segments:
+        return f"{package_name}.endpoints"
+    return ".".join([package_name, "endpoints", *segments])
+
+
+def group_path_from_expr(expr: Any) -> str:
+    plain = get_plain_value(expr)
+    if isinstance(plain, dict) and plain.get("kind") == "ref":
+        parts = [part["value"] for part in plain.get("parts", []) if part.get("kind") == "name"]
+        if not parts:
+            return ""
+        if parts[0] == "GROUPS":
+            parts = parts[1:]
+        return ".".join(parts)
+    if isinstance(plain, str):
+        return plain.strip()
+    if isinstance(plain, list):
+        return ".".join(str(part).strip() for part in plain if str(part).strip())
+    return str(plain).strip() if plain is not None else ""
+
+
+def script_module_name_from_path(script_path: str) -> str:
+    normalized = normalize_script_path(script_path)
+    if normalized.endswith(".py"):
+        normalized = normalized[:-3]
+    normalized = normalized.strip("/")
+    return f"pipelines.{normalized.replace('/', '.')}" if normalized else "pipelines.warmup"
+
+
+def parse_script_reference(expr: Any) -> dict[str, str] | None:
+    plain = get_plain_value(expr)
+    if not isinstance(plain, str):
+        return None
+    script_path, separator, function_name = plain.rpartition(":")
+    if not separator:
+        return None
+    script_path = script_path.strip()
+    function_name = function_name.strip()
+    if not script_path or not function_name:
+        return None
+    return {
+        "path": script_path,
+        "module": script_module_name_from_path(script_path),
+        "function": function_name,
+    }

msra_codegen/docs_generator.py

@@ -0,0 +1,346 @@
+from __future__ import annotations
+
+import shutil
+import textwrap
+from datetime import date
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+from .codegen_context import build_group_context
+from .core_naming import group_public_import_path, root_client_class_name
+from .file_utils import write_text
+from .generator_config import config_section
+from .logo_assets import build_logo_assets
+from .python_formatting import format_python_source
+from .project_model import top_level_groups
+from .readme_pipeline import build_readme_pipeline_code, build_readme_pipeline_note
+from .template_engine import render_template
+
+
+def generate_docs_project(
+    project: dict[str, Any],
+    output_dir: Path,
+    package_name: str,
+    group_tree: dict[str, Any],
+    *,
+    tests_context: dict[str, Any] | None = None,
+) -> None:
+    docs_root = output_dir / "docs"
+    if docs_root.exists():
+        shutil.rmtree(docs_root)
+
+    docs_source_root = docs_root / "source"
+    api_root = docs_source_root / "_api"
+    static_root = docs_source_root / "_static"
+    templates_root = docs_source_root / "_templates"
+    api_root.mkdir(parents=True, exist_ok=True)
+    static_root.mkdir(parents=True, exist_ok=True)
+    templates_root.mkdir(parents=True, exist_ok=True)
+
+    project_source_root = Path(project["source_path"]).resolve().parent
+    logo_context = build_logo_assets(project, project_source_root, static_root)
+    context = build_docs_project_context(
+        project,
+        package_name,
+        group_tree,
+        tests_context=tests_context,
+        logo_context=logo_context,
+    )
+
+    write_text(
+        output_dir / "README.md",
+        render_template("README.md.tpl", context),
+    )
+    write_text(
+        output_dir / "example.py",
+        render_template("example.py.tpl", context),
+    )
+    stale_examples_dir = output_dir / "examples"
+    if stale_examples_dir.exists():
+        shutil.rmtree(stale_examples_dir)
+    write_text(
+        docs_root / "requirements.txt",
+        render_template("docs/requirements.txt.tpl", context),
+    )
+    write_text(
+        docs_source_root / "Makefile",
+        render_template("docs/source/Makefile.tpl", context),
+    )
+    write_text(
+        docs_source_root / "conf.py",
+        render_template("docs/source/conf.py.tpl", context),
+    )
+    write_text(
+        docs_source_root / "index.rst",
+        render_template("docs/source/index.rst.tpl", context),
+    )
+    write_text(
+        docs_source_root / "quick_start.rst",
+        render_template("docs/source/quick_start.rst.tpl", context),
+    )
+    write_text(
+        docs_source_root / "api.rst",
+        render_template("docs/source/api.rst.tpl", context),
+    )
+    write_text(
+        api_root / f"{package_name}.manager.rst",
+        render_template("docs/source/module.rst.tpl", context["manager_module"]),
+    )
+    write_text(
+        api_root / f"{package_name}.endpoints.rst",
+        render_template("docs/source/module.rst.tpl", context["endpoints_module"]),
+    )
+
+    for group_node in top_level_groups(group_tree):
+        write_group_docs(group_node, project, package_name, api_root)
+
+
+def write_group_docs(
+    group_node: dict[str, Any],
+    project: dict[str, Any],
+    package_name: str,
+    api_root: Path,
+) -> None:
+    context = build_group_docs_context(group_node, project, package_name)
+    write_text(
+        api_root / f"{context['import_path']}.rst",
+        render_template("docs/source/module.rst.tpl", context),
+    )
+    for child_node in group_node.get("children", {}).values():
+        write_group_docs(child_node, project, package_name, api_root)
+
+
+def build_docs_project_context(
+    project: dict[str, Any],
+    package_name: str,
+    group_tree: dict[str, Any],
+    *,
+    tests_context: dict[str, Any] | None = None,
+    logo_context: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    app = project["app"]
+    docs_descriptions = config_section("docs", "descriptions")
+    client_class_name = root_client_class_name(project)
+    top_groups = [
+        build_group_docs_context(group_node, project, package_name)
+        for group_node in top_level_groups(group_tree)
+    ]
+    project_title = str(app["name"] or package_name)
+    index_title = f"{project_title} documentation"
+    pipeline_script_code = format_python_source(
+        build_readme_pipeline_code(project, package_name, client_class_name),
+    )
+    return {
+        "title": index_title,
+        "title_underline": "=" * len(index_title),
+        "project_name": project_title,
+        "project_version": str(app["version"]),
+        "package_name": package_name,
+        "logo": logo_context,
+        "browser": str(app.get("browser", "")),
+        "client_class_name": client_class_name,
+        "current_year": str(date.today().year),
+        "manager_module": build_module_page_context(
+            title=f"{package_name}.manager",
+            import_path=f"{package_name}.manager",
+            description=str(docs_descriptions.get("manager") or ""),
+            class_names=[client_class_name, "Warmup"],
+            child_pages=[],
+        ),
+        "endpoints_module": build_module_page_context(
+            title=f"{package_name}.endpoints",
+            import_path=f"{package_name}.endpoints",
+            description=str(docs_descriptions.get("endpoints") or ""),
+            class_names=[group["class_name"] for group in top_groups],
+            child_pages=[group["import_path"] for group in top_groups],
+        ),
+        "api_docnames": [f"_api/{package_name}.endpoints", f"_api/{package_name}.manager"],
+        "top_groups": top_groups,
+        "quick_start": {
+            "title": "Quick Start",
+            "title_underline": "=" * len("Quick Start"),
+            "package_name": package_name,
+            "client_class_name": client_class_name,
+            "requires_camoufox": str(app.get("browser", "")) == "camoufox",
+            "top_groups": top_groups,
+        },
+        "tests": build_readme_tests_context(project, package_name, tests_context),
+        "readme": build_readme_context(
+            project,
+            package_name,
+            pipeline_script_code,
+        ),
+        "pipeline_script_code": pipeline_script_code,
+        "pipeline_script_code_rst": textwrap.indent(pipeline_script_code, "    "),
+        "pipeline_note": build_readme_pipeline_note(project),
+    }
+
+
+def build_readme_context(
+    project: dict[str, Any],
+    package_name: str,
+    pipeline_script_code: str,
+) -> dict[str, Any]:
+    app = project["app"]
+    readme_config = config_section("readme")
+    package_owner = str(app.get("package_owner") or package_name).strip() or package_name
+    package_owner_lower = package_owner.lower()
+    package_name_slug = package_name.replace("_", "-")
+    repo_url = f"https://github.com/{package_owner}/{package_name}"
+    docs_url = f"https://{package_owner_lower}.github.io/{package_name}/quick_start"
+    workflow_url = f"{repo_url}/actions/workflows/tests.yml"
+    workflow_runs_url = f"https://api.github.com/repos/{package_owner}/{package_name}/actions/workflows/tests.yml/runs?per_page=1&status=completed"
+    display_title = str(app.get("name") or package_name).strip() or package_name
+    description = str(app.get("description") or "").strip()
+    badges = build_readme_badges()
+    socials = build_readme_social_links(app.get("social"))
+    return {
+        "title": display_title,
+        "project_line": display_title,
+        "description": description,
+        "package_owner": package_owner,
+        "package_owner_lower": package_owner_lower,
+        "package_name": package_name,
+        "package_name_slug": package_name_slug,
+        "repo_url": repo_url,
+        "issues_url": f"{repo_url}/issues",
+        "docs_url": docs_url,
+        "workflow_url": workflow_url,
+        "workflow_badge_url": f"{workflow_url}/badge.svg",
+        "workflow_last_run_badge_url": (
+            "https://img.shields.io/badge/dynamic/json?label=Tests%20last%20run"
+            "&query=%24.workflow_runs%5B0%5D.updated_at"
+            f"&url={quote(workflow_runs_url, safe='')}"
+            "&logo=githubactions&cacheSeconds=300"
+        ),
+        "pypi_project_url": f"https://pypi.org/project/{package_name_slug}/",
+        "pypi_python_badge_url": f"https://img.shields.io/pypi/pyversions/{package_name}",
+        "pypi_version_badge_url": f"https://img.shields.io/pypi/v/{package_name}?color=blue",
+        "pypi_downloads_badge_url": f"https://img.shields.io/pypi/dm/{package_name}?label=PyPi%20downloads",
+        "license_badge_url": f"https://img.shields.io/github/license/{package_owner}/{package_name}",
+        "license_url": f"{repo_url}/blob/main/LICENSE",
+        "badges": badges,
+        "socials": socials,
+        "principle_text": str(readme_config.get("principle_text", "Библиотека полностью повторяет сетевую работу обычного пользователя на сайте.")),
+        "pipeline_script_code": pipeline_script_code,
+    }
+
+
+def build_readme_tests_context(
+    project: dict[str, Any],
+    package_name: str,
+    tests_context: dict[str, Any] | None,
+) -> dict[str, Any]:
+    client_class_name = root_client_class_name(project)
+    has_autotests = False
+    if isinstance(tests_context, dict):
+        api_test = tests_context.get("api_test")
+        if isinstance(api_test, dict):
+            has_autotests = any(bool(api_test.get(key)) for key in ("hooks", "providers", "manual_tests", "data_cases"))
+    return {
+        "has_autotests": has_autotests,
+        "autotest_start_class": f"{package_name}.{client_class_name}",
+    }
+
+
+def build_readme_social_links(social: Any) -> list[dict[str, Any]]:
+    if not isinstance(social, dict):
+        return []
+    readme_socials = config_section("readme").get("socials", [])
+    links: list[dict[str, Any]] = []
+    if not isinstance(readme_socials, list):
+        return links
+    for item in readme_socials:
+        if not isinstance(item, dict):
+            continue
+        key = str(item.get("key") or "").strip()
+        if not key:
+            continue
+        url = str(social.get(key) or "").strip()
+        if not url:
+            continue
+        badge_url = str(item.get("badge_url", "")).strip()
+        if not badge_url:
+            continue
+        links.append(
+            {
+                "key": key,
+                "label": str(item.get("label", key.title())),
+                "badge_url": badge_url,
+                "url": url,
+            }
+        )
+    return links
+
+
+def build_readme_badges() -> list[dict[str, Any]]:
+    readme_badges = config_section("readme").get("badges", [])
+    if not isinstance(readme_badges, list):
+        return []
+    badges: list[dict[str, Any]] = []
+    for item in readme_badges:
+        if not isinstance(item, dict):
+            continue
+        label = str(item.get("label", "")).strip()
+        badge_url = str(item.get("badge_url", "")).strip()
+        url = str(item.get("url", "")).strip()
+        if not label or not badge_url or not url:
+            continue
+        badges.append(
+            {
+                "label": label,
+                "badge_url": badge_url,
+                "url": url,
+            }
+        )
+    return badges
+
+
+def build_group_docs_context(
+    group_node: dict[str, Any],
+    project: dict[str, Any],
+    package_name: str,
+) -> dict[str, Any]:
+    group_context = build_group_context(group_node, project, package_name)
+    child_nodes = list(group_node.get("children", {}).values())
+    child_class_names = [child["class_name"] for child in group_context["child_imports"]]
+    class_names = [group_context["class_name"], *child_class_names]
+    import_path = group_public_import_path(package_name, group_node.get("path", []))
+    context = build_module_page_context(
+        title=import_path,
+        import_path=import_path,
+        description=group_context["description"],
+        class_names=list(dict.fromkeys(class_names)),
+        child_pages=[group_public_import_path(package_name, child["path"]) for child in child_nodes],
+    )
+    context.update(
+        {
+            "class_name": group_context["class_name"],
+            "child_class_names": list(dict.fromkeys(child_class_names)),
+            "field_name": group_node["path"][-1] if group_node.get("path") else "Group",
+        }
+    )
+    return context
+
+
+def build_module_page_context(
+    *,
+    title: str,
+    import_path: str,
+    description: str,
+    class_names: list[str],
+    child_pages: list[str],
+) -> dict[str, Any]:
+    class_names = list(dict.fromkeys(class_names))
+    child_pages = list(dict.fromkeys(child_pages))
+    return {
+        "title": title,
+        "title_underline": "=" * len(title),
+        "import_path": import_path,
+        "description": description,
+        "class_names": class_names,
+        "class_refs": [f"{import_path}.{name}" for name in class_names],
+        "child_pages": child_pages,
+        "child_docnames": child_pages,
+    }

msra_codegen/file_utils.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def write_text(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")