aas-core-codegen 0.0.16__py3-none-any.whl

dev/continuous_integration/precommit.py

@@ -0,0 +1,400 @@
+#!/usr/bin/env python3
+
+"""Run pre-commit checks on the repository."""
+import argparse
+import enum
+import os
+import pathlib
+import shlex
+import subprocess
+import sys
+from typing import Mapping, Optional, Sequence
+
+# pylint: disable=unnecessary-comprehension
+
+
+class Step(enum.Enum):
+    """Enumerate different pre-commit steps."""
+
+    REFORMAT = "reformat"
+    ISORT = "isort"
+    SSORT = "ssort"
+    MYPY = "mypy"
+    PYLINT = "pylint"
+    TEST = "test"
+    DOCTEST = "doctest"
+    CHECK_INIT_AND_PYPROJECT_CONSISTENT = "check-init-and-pyproject-consistent"
+    CHECK_HELP_IN_README = "check-help-in-readme"
+    INTEGRATION_TEST = "integration-test"
+
+
+def call_and_report(
+    verb: str,
+    cmd: Sequence[str],
+    cwd: Optional[pathlib.Path] = None,
+    env: Optional[Mapping[str, str]] = None,
+) -> int:
+    """
+    Wrap a subprocess call with the reporting to STDERR if it failed.
+
+    Return 1 if there is an error and 0 otherwise.
+    """
+    cmd_str = " ".join(shlex.quote(part) for part in cmd)
+
+    if cwd is not None:
+        print(f"Executing from {cwd}: {cmd_str}")
+    else:
+        print(f"Executing: {cmd_str}")
+
+    exit_code = subprocess.call(cmd, cwd=str(cwd) if cwd is not None else None, env=env)
+
+    if exit_code != 0:
+        print(
+            f"Failed to {verb} with exit code {exit_code}: {cmd_str}", file=sys.stderr
+        )
+
+    return exit_code
+
+
+def main() -> int:
+    """Execute entry_point routine."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--overwrite",
+        help="Try to automatically fix the offending files (e.g., by re-formatting).",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--select",
+        help=(
+            "If set, only the selected steps are executed. "
+            "This is practical if some of the steps failed and you want to "
+            "fix them in isolation. "
+            "The steps are given as a space-separated list of: "
+            + " ".join(value.value for value in Step)
+        ),
+        metavar="",
+        nargs="+",
+        choices=[value.value for value in Step],
+    )
+    parser.add_argument(
+        "--skip",
+        help=(
+            "If set, skips the specified steps. "
+            "This is practical if some of the steps passed and "
+            "you want to fix the remainder in isolation. "
+            "The steps are given as a space-separated list of: "
+            + " ".join(value.value for value in Step)
+        ),
+        metavar="",
+        nargs="+",
+        choices=[value.value for value in Step],
+    )
+
+    args = parser.parse_args()
+
+    overwrite = bool(args.overwrite)
+
+    selects = (
+        [Step(value) for value in args.select]
+        if args.select is not None
+        else [value for value in Step]
+    )
+    skips = [Step(value) for value in args.skip] if args.skip is not None else []
+
+    repo_root = pathlib.Path(os.path.realpath(__file__)).parent.parent.parent
+
+    if Step.REFORMAT in selects and Step.REFORMAT not in skips:
+        print("Re-formatting...")
+        reformat_targets = [
+            "aas_core_codegen",
+            "dev/continuous_integration",
+            "dev/dev_scripts",
+            "dev/integration_tests",
+            "dev/tests",
+            "dev/test_data",
+        ]
+
+        # NOTE (mristin, 2022-10-29):
+        # We exclude the snippets and the expected output from the formatting as
+        # this causes too many unpredictable results and thus makes the generator
+        # harder to debug.
+        exclude = ["dev/test_data/python", "dev/test_data/python_protobuf"]
+
+        if overwrite:
+            exit_code = call_and_report(
+                verb="black",
+                cmd=[sys.executable, "-m", "black"]
+                + reformat_targets
+                + ["--exclude"]
+                + exclude,
+                cwd=repo_root,
+            )
+            if exit_code != 0:
+                return 1
+        else:
+            exit_code = call_and_report(
+                verb="check with black",
+                cmd=[sys.executable, "-m", "black", "--check"]
+                + reformat_targets
+                + ["--exclude"]
+                + exclude,
+                cwd=repo_root,
+            )
+            if exit_code != 0:
+                return 1
+    else:
+        print("Skipped re-formatting.")
+
+    if Step.SSORT in selects and Step.SSORT not in skips:
+        print("Ssort'ing...")
+
+        ssort_targets = [
+            "aas_core_codegen",
+            "dev/continuous_integration",
+            "dev/dev_scripts",
+            "dev/tests",
+        ]
+
+        if overwrite:
+            exit_code = call_and_report(
+                verb="ssort",
+                cmd=[sys.executable, "-m", "ssort"] + ssort_targets,
+                cwd=repo_root,
+            )
+            if exit_code != 0:
+                return 1
+
+            # NOTE (mristin, 2024-01-26):
+            # We have to re-format just after the ssort,
+            # see: https://pypi.org/project/ssort/,
+            # "We recommend that you reformat using isort and black
+            # immediately after running ssort."
+            exit_code = call_and_report(
+                verb="black after ssort",
+                cmd=[sys.executable, "-m", "black"] + ssort_targets,
+                cwd=repo_root,
+            )
+            if exit_code != 0:
+                return 1
+        else:
+            exit_code = call_and_report(
+                verb="check with ssort",
+                cmd=[sys.executable, "-m", "ssort", "--check", "--diff"]
+                + ssort_targets,
+                cwd=repo_root,
+            )
+            if exit_code != 0:
+                return 1
+
+    if Step.MYPY in selects and Step.MYPY not in skips:
+        print("Mypy'ing...")
+        mypy_targets = [
+            "aas_core_codegen",
+            "dev/tests",
+            "dev/continuous_integration",
+            "dev/dev_scripts",
+        ]
+        config_file = pathlib.Path("dev/continuous_integration") / "mypy.ini"
+
+        exit_code = call_and_report(
+            verb="mypy",
+            cmd=[
+                sys.executable,
+                "-m",
+                "mypy",
+                "--strict",
+                "--config-file",
+                str(config_file),
+            ]
+            + mypy_targets,
+            cwd=repo_root,
+        )
+        if exit_code != 0:
+            return 1
+    else:
+        print("Skipped mypy'ing.")
+
+    if Step.PYLINT in selects and Step.PYLINT not in skips:
+        print("Pylint'ing...")
+        pylint_targets = [
+            "aas_core_codegen",
+            "dev/tests",
+            "dev/continuous_integration",
+            "dev/dev_scripts",
+        ]
+        rcfile = pathlib.Path("dev/continuous_integration") / "pylint.rc"
+
+        exit_code = call_and_report(
+            verb="pylint",
+            cmd=[sys.executable, "-m", "pylint", f"--rcfile={rcfile}"] + pylint_targets,
+            cwd=repo_root,
+        )
+        if exit_code != 0:
+            return 1
+    else:
+        print("Skipped pylint'ing.")
+
+    if Step.TEST in selects and Step.TEST not in skips:
+        print("Testing...")
+        env = os.environ.copy()
+        env["ICONTRACT_SLOW"] = "true"
+
+        exit_code = call_and_report(
+            verb="execute unit tests",
+            cmd=[
+                sys.executable,
+                "-m",
+                "coverage",
+                "run",
+                "--source",
+                "aas_core_codegen",
+                "-m",
+                "unittest",
+                "discover",
+                "-s",
+                "dev/tests",
+            ],
+            cwd=repo_root,
+            env=env,
+        )
+        if exit_code != 0:
+            return 1
+
+        exit_code = call_and_report(
+            verb="report the coverage", cmd=["coverage", "report"]
+        )
+        if exit_code != 0:
+            return 1
+    else:
+        print("Skipped testing.")
+
+    if Step.INTEGRATION_TEST in selects and Step.INTEGRATION_TEST not in skips:
+        print("Integration testing...")
+        exit_code = call_and_report(
+            verb="execute unit tests",
+            cmd=[
+                sys.executable,
+                "dev/integration_tests/main.py",
+            ],
+            cwd=repo_root,
+        )
+        if exit_code != 0:
+            return 1
+    else:
+        print("Skipped integration testing.")
+
+    if Step.DOCTEST in selects and Step.DOCTEST not in skips:
+        print("Doctest'ing...")
+
+        # BEFORE-RELEASE (mristin, 2021-12-13):
+        #  Add ``{repo_root}/docs/source/**/*.rst`` as well here
+        doc_files = ["README.rst"]
+
+        exit_code = call_and_report(
+            verb="doctest",
+            cmd=[sys.executable, "-m", "doctest"] + doc_files,
+            cwd=repo_root,
+        )
+        if exit_code != 0:
+            return 1
+
+        for pth in (repo_root / "aas_core_codegen").glob("**/*.py"):
+            if pth.name == "__main__.py":
+                continue
+
+            # NOTE (mristin, 2021-12-27):
+            # The subprocess calls are expensive, call only if there is an actual
+            # doctest
+            text = pth.read_text(encoding="utf-8")
+            if ">>>" in text:
+                exit_code = call_and_report(
+                    verb="doctest",
+                    cmd=[sys.executable, "-m", "doctest", str(pth)],
+                    cwd=repo_root,
+                )
+                if exit_code != 0:
+                    return 1
+    else:
+        print("Skipped doctest'ing.")
+
+    if (
+        Step.CHECK_INIT_AND_PYPROJECT_CONSISTENT in selects
+        and Step.CHECK_INIT_AND_PYPROJECT_CONSISTENT not in skips
+    ):
+        print(
+            "Checking that aas_core_codegen/__init__.py and pyproject.toml "
+            "are consistent..."
+        )
+        exit_code = call_and_report(
+            verb=(
+                "check that aas_core_codegen/__init__.py and pyproject.toml "
+                "are consistent"
+            ),
+            cmd=[
+                sys.executable,
+                "dev/continuous_integration/check_init_and_pyproject_consistent.py",
+            ],
+            cwd=repo_root,
+        )
+        if exit_code != 0:
+            return 1
+    else:
+        print(
+            "Skipped checking that aas_core_codegen/__init__.py and "
+            "pyproject.toml are consistent."
+        )
+
+    # NOTE (mristin):
+    # We need to check for the Python version since ``argparse`` output changes
+    # between the versions. Hence, we pin it at the moment to Python 3.12.
+
+    if (3, 12) <= sys.version_info < (3, 13):
+        if (
+            Step.CHECK_HELP_IN_README in selects
+            and Step.CHECK_HELP_IN_README not in skips
+        ):
+            cmd = [sys.executable, "dev/continuous_integration/check_help_in_readme.py"]
+            if overwrite:
+                cmd.append("--overwrite")
+
+            if not overwrite:
+                print("Checking that --help's and the readme coincide...")
+                exit_code = call_and_report(
+                    verb="check that --help's and the readme coincide",
+                    cmd=[
+                        sys.executable,
+                        "dev/continuous_integration/check_help_in_readme.py",
+                    ],
+                    cwd=repo_root,
+                )
+                if exit_code != 0:
+                    return 1
+            else:
+                print("Overwriting the --help's in the readme...")
+                exit_code = call_and_report(
+                    verb="overwrite the --help's in the readme",
+                    cmd=[
+                        sys.executable,
+                        "dev/continuous_integration/check_help_in_readme.py",
+                        "--overwrite",
+                    ],
+                    cwd=repo_root,
+                )
+                if exit_code != 0:
+                    return 1
+
+            subprocess.check_call(cmd, cwd=str(repo_root))
+        else:
+            print("Skipped checking that --help's and the doc coincide.")
+    else:
+        print(
+            "Skipped checking that --help's and the doc coincide "
+            "since we pin it on Python version 3.12."
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev/dev_scripts/__init__.py

@@ -0,0 +1 @@
+"""Provide scripts which are manually invoked during the development."""

dev/dev_scripts/compare_rendered_regexes_against_source_py.py

@@ -0,0 +1,42 @@
+"""Compare the rendered regexes against their original form in ``source.py``."""
+
+import os
+import pathlib
+import sys
+from typing import Optional
+
+
+def main() -> int:
+    """Execute the main routine."""
+
+    repo_dir = pathlib.Path(os.path.realpath(__file__)).parent.parent.parent
+    test_cases_dir = repo_dir / "dev/test_data/parse_retree"
+    expected_pths = sorted((test_cases_dir / "expected").glob("**/source.py"))
+
+    for pth in expected_pths:
+        rendered_regex_pth = pth.parent / "rendered_regex.txt"
+
+        source = pth.read_text(encoding="utf-8").strip()
+
+        source_stem = None  # type: Optional[str]
+        if source.startswith('"') and source.endswith('"'):
+            source_stem = source[1:-1]
+        elif source.startswith('f"') and source.endswith('"'):
+            source_stem = source[2:-1]
+        else:
+            pass
+
+        rendered = rendered_regex_pth.read_text(encoding="utf-8")
+        rendered_repr_stem = repr(rendered)[1:-1]
+
+        if source_stem is None or rendered_repr_stem != source_stem:
+            print()
+            print(str(pth))
+            print(source)
+            print(repr(rendered))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev/dev_scripts/copy_to_aas_core3_cpp.py

@@ -0,0 +1,100 @@
+"""
+Copy the generated C++ files for AAS v3.0 to the corresponding C++ repository.
+
+This script is practical as a part of the development pipeline:
+1) Re-generate C++ files (through re-recording the test data),
+2) Copy the generated files (with this script),
+3) Run clang-format and clang-tidy from the C++ repository, and
+4) Run the unit tests from the C++ repository.
+"""
+
+import argparse
+import os
+import pathlib
+import shutil
+import sys
+
+
+def main() -> int:
+    """Execute the main routine."""
+    repo_root = pathlib.Path(os.path.realpath(__file__)).parent.parent.parent
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--aas_core3_cpp_path",
+        help="Path to the aas-core3.0-cpp repository",
+        default=str(repo_root.parent / "aas-core3.0-cpp"),
+    )
+    args = parser.parse_args()
+
+    aas_core3_cpp_path = pathlib.Path(args.aas_core3_cpp_path)
+
+    if not aas_core3_cpp_path.exists():
+        print(
+            f"--aas_core3_cpp_path does not exist: {aas_core3_cpp_path}",
+            file=sys.stderr,
+        )
+        return 1
+
+    if not aas_core3_cpp_path.is_dir():
+        print(
+            f"--aas_core3_cpp_path is not a directory: {aas_core3_cpp_path}",
+            file=sys.stderr,
+        )
+        return 1
+
+    expected_output_dir = (
+        repo_root
+        / "dev"
+        / "test_data"
+        / "cpp"
+        / "test_main"
+        / "aas_core_meta.v3"
+        / "expected_output"
+    )
+
+    assert aas_core3_cpp_path.exists()
+
+    target_src_dir = aas_core3_cpp_path / "src"
+    target_src_dir.mkdir(exist_ok=True)
+
+    target_include_dir = aas_core3_cpp_path / "include" / "aas_core" / "aas_3_0"
+    target_include_dir.mkdir(exist_ok=True, parents=True)
+
+    paths = sorted(
+        list(expected_output_dir.glob("*.cpp"))
+        + list(expected_output_dir.glob("*.hpp"))
+    )
+
+    print(
+        f"Copying *.cpp and *.hpp\n"
+        f"  from {expected_output_dir}\n"
+        f"  to {target_src_dir} and {target_include_dir}..."
+    )
+    for pth in paths:
+        if pth.suffix == ".cpp":
+            dst = target_src_dir / pth.name
+        elif pth.suffix == ".hpp":
+            dst = target_include_dir / pth.name
+        else:
+            raise AssertionError(f"Unhandled suffix: {pth.suffix} from: {pth}")
+
+        # NOTE (mristin, 2024-01-08):
+        # We compare the contents to avoid re-compilations by CMake.
+        should_copy = True
+        if dst.exists():
+            src_text = pth.read_text(encoding="utf-8")
+            dst_text = dst.read_text(encoding="utf-8")
+
+            should_copy = src_text != dst_text
+
+        if should_copy:
+            shutil.copy(src=pth, dst=dst)
+
+    print("Copied.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev/dev_scripts/copy_to_aas_core3_java.py

@@ -0,0 +1,90 @@
+"""
+Copy the generated Java files for AAS v3.0 to the corresponding Java repository.
+
+This script is practical as a part of the development pipeline:
+1) Re-generate Java files (through re-recording the test data),
+2) Copy the generated files (with this script),
+3) Run the checks and reformatting from the Java repository, and
+4) Re-generate the javadoc.
+"""
+
+import argparse
+import os
+import pathlib
+import shutil
+import sys
+
+
+def main() -> int:
+    """Execute the main routine."""
+    repo_root = pathlib.Path(os.path.realpath(__file__)).parent.parent.parent
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--aas_core3_java_path",
+        help="Path to the aas-core3.0-java repository",
+        default=str(repo_root.parent / "aas-core3.0-java"),
+    )
+    args = parser.parse_args()
+
+    aas_core3_java_path = pathlib.Path(args.aas_core3_java_path)
+
+    if not aas_core3_java_path.exists():
+        print(
+            f"--aas_core3_java_path does not exist: {aas_core3_java_path}",
+            file=sys.stderr,
+        )
+        return 1
+
+    if not aas_core3_java_path.is_dir():
+        print(
+            f"--aas_core3_java_path is not a directory: {aas_core3_java_path}",
+            file=sys.stderr,
+        )
+        return 1
+
+    expected_output_dir = (
+        repo_root
+        / "dev"
+        / "test_data"
+        / "java"
+        / "test_main"
+        / "aas_core_meta.v3"
+        / "expected_output"
+    )
+
+    assert aas_core3_java_path.exists()
+
+    target_src_dir = aas_core3_java_path / "src/main/java/aas_core/aas3_0"
+    target_src_dir.mkdir(exist_ok=True)
+
+    paths = sorted(expected_output_dir.glob("**/*.java"))
+
+    print(
+        f"Copying *.java\n"
+        f"  from {expected_output_dir}\n"
+        f"  to {target_src_dir}..."
+    )
+    for pth in paths:
+        dst = target_src_dir / pth.relative_to(expected_output_dir)
+
+        # NOTE (mristin):
+        # We compare the contents to avoid re-compilations.
+        should_copy = True
+        if dst.exists():
+            src_text = pth.read_text(encoding="utf-8")
+            dst_text = dst.read_text(encoding="utf-8")
+
+            should_copy = src_text != dst_text
+
+        if should_copy:
+            print(f"    Copying {pth} to {dst}...")
+            shutil.copy(src=pth, dst=dst)
+
+    print("Copied.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())