aas-core-codegen 0.0.16__py3-none-any.whl

dev/dev_scripts/download_latest_aas_core_meta_v3.py

@@ -0,0 +1,114 @@
+"""Download the latest meta-model for V3.0 to the test data."""
+
+import argparse
+import os
+import pathlib
+import sys
+from typing import Final, Mapping, Union
+
+import black
+import requests
+
+OWNER: Final[str] = "aas-core-works"
+REPO: Final[str] = "aas-core-meta"
+REF: Final[str] = "main"
+REMOTE_PATH: Final[str] = "aas_core_meta/v3.py"
+
+
+def _latest_commit_sha(timeout: float = 15.0) -> str:
+    """Resolve the latest commit SHA on the ``REF`` for a specific path."""
+    params: Mapping[str, Union[str, int]] = {
+        "path": REMOTE_PATH,
+        "sha": REF,
+        "per_page": 1,
+    }
+    resp = requests.get(
+        f"https://api.github.com/repos/{OWNER}/{REPO}/commits",
+        params=params,
+        timeout=timeout,
+    )
+    try:
+        resp.raise_for_status()
+    except requests.HTTPError as ex:
+        raise RuntimeError(
+            f"Failed to resolve latest commit for "
+            f"{OWNER}/{REPO}:{REF} path={REMOTE_PATH} "
+            f"({resp.status_code}): {resp.text}"
+        ) from ex
+
+    commits = resp.json()
+    if not commits or not isinstance(commits, list):
+        raise RuntimeError(
+            "Could not determine latest commit SHA (empty API response)."
+        )
+
+    sha = commits[0].get("sha")
+    if not isinstance(sha, str):
+        raise RuntimeError("API did not return a valid commit SHA.")
+
+    if len(sha) == 0:
+        raise RuntimeError("API returned an empty commit SHA.")
+
+    return sha
+
+
+def _raw_url(sha: str) -> str:
+    """Get the URL of the raw file on GitHub."""
+    return f"https://raw.githubusercontent.com/{OWNER}/{REPO}/{sha}/{REMOTE_PATH}"
+
+
+def _download(url: str, timeout: float = 30.0) -> str:
+    """Download the raw file contents."""
+    resp = requests.get(url, timeout=timeout)
+    try:
+        resp.raise_for_status()
+    except requests.HTTPError as ex:
+        raise RuntimeError(
+            f"Failed to download file from {url} ({resp.status_code}): {resp.text}"
+        ) from ex
+
+    return resp.text
+
+
+def retrieve_sha_and_download() -> str:
+    """
+    Get the latest SHA revision of the meta-model and download it to test data.
+
+    Return the SHA of the downloaded meta-model.
+    """
+    sha = _latest_commit_sha()
+
+    raw_url = _raw_url(sha)
+
+    content = _download(url=raw_url)
+
+    banner = f"# Downloaded from: {raw_url}\n# Do NOT edit or append!"
+
+    repo_root = pathlib.Path(os.path.realpath(__file__)).parent.parent.parent
+
+    out_path = repo_root / "dev/test_data/real_meta_models/aas_core_meta.v3.py"
+    out_path.parent.mkdir(exist_ok=True)
+
+    out_path.write_text(f"{banner}\n\n{content.rstrip()}\n\n{banner}", encoding="utf-8")
+
+    black.format_file_in_place(
+        src=out_path, fast=False, mode=black.FileMode(), write_back=black.WriteBack.YES
+    )
+
+    print(f"Wrote and reformatted to: {out_path} (from commit {sha[:8]}).")
+
+    return sha
+
+
+def main() -> int:
+    """Execute the main routine."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    _ = parser.parse_args()
+
+    retrieve_sha_and_download()
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev/dev_scripts/draw_bipartite_graph_based_on_lines.py

@@ -0,0 +1,37 @@
+"""
+Draw a bipartite graph on matching lines in two files.
+
+This is helpful when you want to minimize the diffs. Unfortunately, ``diff`` is not
+really a good tool when it comes to re-shuffling of the blocks.
+"""
+
+import argparse
+import pathlib
+import sys
+
+
+def main() -> int:
+    """Execute the main routine."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--ours", help="path to the first file", required=True)
+    parser.add_argument("--theirs", help="path to the second file", required=True)
+    args = parser.parse_args()
+
+    ours_pth = pathlib.Path(args.ours)
+    theirs_pth = pathlib.Path(args.theirs)
+
+    our_lines = ours_pth.read_text(encoding="utf-8").splitlines()
+    their_lines = theirs_pth.read_text(encoding="utf-8").splitlines()
+
+    their_lines_by_index = {line: i for i, line in enumerate(their_lines)}
+
+    for i, line in enumerate(our_lines):
+        their_i = their_lines_by_index.get(line, None)
+        if their_i is not None:
+            print(f"Ours {i+1:4d} to theirs {their_i:4d}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev/dev_scripts/run_tests_with_rerecord.py

@@ -0,0 +1,69 @@
+"""Run all the tests with the environment variable ``AAS_CORE_CODEGEN_RERECORD`` set."""
+
+import argparse
+import os
+import pathlib
+import subprocess
+import sys
+import shlex
+
+
+def main() -> int:
+    """Execute the main routine."""
+    available_tests = [
+        "tests.csharp.test_main.Test_against_recorded",
+        "tests.csharp.test_verification.Test_against_recorded",
+        "tests.csharp.test_structure.Test_generation_against_recorded",
+        "tests.intermediate.test_translate.Test_against_recorded",
+        "tests.java.test_main.Test_against_recorded",
+        "tests.our_jsonschema.test_main.Test_against_recorded",
+        "tests.rdf_shacl.test_main.Test_against_recorded",
+        "tests.parse.test_parse.Test_against_recorded",
+        "tests.parse.test_retree.Test_against_recorded",
+        "tests.python.test_main.Test_against_recorded",
+        "tests.smoke.test_main.Test_against_recorded",
+        "tests.typescript.test_main.Test_against_recorded",
+        "tests.xsd.test_main.Test_against_recorded",
+    ]
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--select",
+        help=(
+            "If set, only the selected tests are executed. "
+            "This is practical if some of the tests failed and you want to "
+            "fix them in isolation. "
+            "The tests are given as a space-separated list of: "
+            + " ".join(available_tests)
+        ),
+        metavar="",
+        nargs="+",
+        choices=available_tests,
+    )
+    args = parser.parse_args()
+
+    repo_root = pathlib.Path(os.path.realpath(__file__)).parent.parent.parent
+
+    env = os.environ.copy()
+    env["AAS_CORE_CODEGEN_RERECORD"] = "1"
+
+    if args.select is not None:
+        tests_to_run = list(args.select)
+    else:
+        tests_to_run = available_tests
+
+    for qualified_test_name in tests_to_run:
+        cmd = [sys.executable, "-m", "unittest", "-v", qualified_test_name]
+
+        print(f"Executing {qualified_test_name}...")
+        exit_code = subprocess.call(cmd, cwd=str(repo_root), env=env)
+        if exit_code != 0:
+            cmd_str = " ".join(shlex.quote(part) for part in cmd)
+            print(f"Failed to execute the test: {cmd_str}")
+            return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev/dev_scripts/update_to_aas_core_meta.py

@@ -0,0 +1,174 @@
+"""
+Update everything in this project to the latest aas-core-meta.
+
+Git is expected to be installed.
+"""
+
+import argparse
+import os
+import pathlib
+import subprocess
+import sys
+import tempfile
+import time
+from typing import Optional
+
+import dev_scripts.download_latest_aas_core_meta_v3
+
+
+def _make_sure_no_changed_files(
+    repo_dir: pathlib.Path, expected_our_branch: str
+) -> Optional[int]:
+    """
+    Make sure that no files are modified in our repository.
+
+    Return exit code if something is unexpected.
+    """
+    diff_name_status = subprocess.check_output(
+        ["git", "diff", "--name-status", expected_our_branch],
+        cwd=str(repo_dir),
+        encoding="utf-8",
+    ).strip()
+
+    if len(diff_name_status.splitlines()) > 0:
+        print(
+            f"The following files are modified "
+            f"compared to branch {expected_our_branch!r} in {repo_dir}:\n"
+            f"{diff_name_status}\n"
+            f"\n"
+            f"Please stash the changes first before you update to aas-core-meta.",
+            file=sys.stderr,
+        )
+        return 1
+
+    return None
+
+
+def _rerecord_everything(repo_dir: pathlib.Path) -> Optional[int]:
+    """
+    Run all unit tests with re-record environment variable set.
+
+    Return an error code, if any.
+    """
+    env = os.environ.copy()
+    env["AAS_CORE_CODEGEN_RERECORD"] = "1"
+
+    starting_points = [
+        pth
+        for pth in (repo_dir / "tests").glob("*")
+        if (
+            pth.is_dir()
+            and not pth.name.startswith("__")
+            and not pth.name.startswith(".")
+        )
+    ]
+
+    start = time.perf_counter()
+
+    for starting_point in starting_points:
+        print(f"Starting to run tests in: {starting_point} ...")
+
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                "unittest",
+                "discover",
+                "--start-directory",
+                str(starting_point),
+            ],
+            env=env,
+            cwd=str(repo_dir),
+        )
+
+    duration = time.perf_counter() - start
+    print(f"Re-recording took: {duration:.2f} seconds.")
+    return None
+
+
+def _create_branch_commit_and_push(
+    repo_dir: pathlib.Path, aas_core_meta_revision: str
+) -> None:
+    """Create a feature branch, commit the changes and push it."""
+    branch = f"Update-test-data-to-aas-core-meta-{aas_core_meta_revision}"
+    print(f"Creating the branch {branch!r}...")
+    subprocess.check_call(["git", "checkout", "-b", branch], cwd=repo_dir)
+
+    print("Adding files...")
+    subprocess.check_call(["git", "add", "."], cwd=repo_dir)
+
+    # pylint: disable=line-too-long
+    message = f"""\
+Update test data to aas-core-meta {aas_core_meta_revision}
+
+We update the development requirements to and re-record the test data
+for [aas-core-meta {aas_core_meta_revision}].
+
+[aas-core-meta {aas_core_meta_revision}]: https://github.com/aas-core-works/aas-core-meta/commit/{aas_core_meta_revision}"""
+    # pylint: enable=line-too-long
+
+    print("Committing...")
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_file = pathlib.Path(tmp_dir) / "commit-message.txt"
+        tmp_file.write_text(message, encoding="utf-8")
+
+        subprocess.check_call(["git", "commit", "--file", str(tmp_file)], cwd=repo_dir)
+
+    print(f"Pushing to remote {branch}...")
+    subprocess.check_call(["git", "push", "-u"], cwd=repo_dir)
+
+
+def main() -> int:
+    """Execute the main routine."""
+    repo_dir = pathlib.Path(os.path.realpath(__file__)).parent.parent.parent
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--expected_our_branch",
+        help="Git branch expected in this repository",
+        default="main",
+    )
+    args = parser.parse_args()
+
+    expected_our_branch = str(args.expected_our_branch)
+
+    our_branch = subprocess.check_output(
+        ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+        cwd=str(repo_dir),
+        encoding="utf-8",
+    ).strip()
+    if our_branch != expected_our_branch:
+        print(
+            f"--expected_our_branch is {expected_our_branch}, "
+            f"but got {our_branch} in: {repo_dir}",
+            file=sys.stderr,
+        )
+        return 1
+
+    exit_code = _make_sure_no_changed_files(
+        repo_dir=repo_dir, expected_our_branch=expected_our_branch
+    )
+    if exit_code is not None:
+        return exit_code
+
+    # fmt: off
+    aas_core_meta_revision = (
+        dev_scripts.download_latest_aas_core_meta_v3.retrieve_sha_and_download()
+    )[:8]
+    # fmt: on
+
+    print(f"The aas-core-meta revision is: {aas_core_meta_revision}")
+
+    exit_code = _rerecord_everything(repo_dir=repo_dir)
+    if exit_code is not None:
+        return exit_code
+
+    _create_branch_commit_and_push(
+        repo_dir=repo_dir, aas_core_meta_revision=aas_core_meta_revision
+    )
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev/integration_tests/input/jsonschema/boilerplate/main.py

@@ -0,0 +1,55 @@
+"""Check the JSON schema against the given JSON file."""
+import argparse
+import json
+import pathlib
+import sys
+import warnings
+
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+    # NOTE (mristin):
+    # We need to disable warnings. Jsonschema package at the latest version (4.4.0) has
+    # a problem with JSON schema draft 2019-09 and crashes with a recursion error,
+    # see: https://github.com/python-jsonschema/jsonschema/issues/847.
+    #
+    # We revert to jsonschema 3.2.0, which can not handle 2019-09, but still seems
+    # to validate correctly our examples.
+    import jsonschema
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--model_path",
+        help="Path to a model to be validated against the generated JSON schema",
+        required=True,
+    )
+    parser.add_argument("--schema_path", help="Path to the JSON schema", required=True)
+    args = parser.parse_args()
+    model_path = pathlib.Path(args.model_path)
+    schema_path = pathlib.Path(args.schema_path)
+
+    if not model_path.exists():
+        print(f"--model_path does not exist: {model_path}", file=sys.stderr)
+        sys.exit(1)
+
+    if not schema_path.exists():
+        print(f"--schema_path does not exist: {schema_path}", file=sys.stderr)
+        sys.exit(1)
+
+    with schema_path.open("rt") as fid:
+        schema = json.load(fid)
+
+    with model_path.open("rt") as fid:
+        instance = json.load(fid)
+
+    try:
+        jsonschema.validate(instance=instance, schema=schema)
+    except jsonschema.ValidationError as err:
+        raise AssertionError(
+            f"Failed to validate {model_path} against {schema_path}"
+        ) from err
+
+
+if __name__ == "__main__":
+    main()

dev/integration_tests/input/meta_model.py

@@ -0,0 +1,38 @@
+from typing import List
+
+
+class My_Class:
+    string_prop: str
+    int_prop: int
+    bool_prop: bool
+
+    def __init__(self, string_prop: str, int_prop: int, bool_prop: bool) -> None:
+        self.string_prop = string_prop
+        self.int_prop = int_prop
+        self.bool_prop = bool_prop
+
+
+class Root:
+    foo: My_Class
+    foos: List[My_Class]
+    strings: List[str]
+    integers: List[int]
+    booleans: List[bool]
+
+    def __init__(
+        self,
+        foo: My_Class,
+        foos: List[My_Class],
+        strings: List[str],
+        integers: List[int],
+        booleans: List[bool],
+    ) -> None:
+        self.foo = foo
+        self.foos = foos
+        self.strings = strings
+        self.integers = integers
+        self.booleans = booleans
+
+
+__version__ = "dummy"
+__xml_namespace__ = "https://dummy.com"

dev/integration_tests/input/python/boilerplate/main.py

@@ -0,0 +1,153 @@
+"""De-serialize the model, validate it and serialize it back."""
+import argparse
+import difflib
+import json
+import pathlib
+import sys
+from xml.dom import minidom
+
+if sys.version_info >= (3, 9):
+    from typing import Literal, Optional, Any
+else:
+    from typing_extensions import Literal
+
+if sys.version_info >= (3, 11):
+    from typing import assert_never
+else:
+    from typing_extensions import assert_never
+
+# The module aas_core is expected to be generated by the integration test.
+from sdk import types, jsonization, xmlization, verification
+
+
+def normalize_xml(xml_text: str) -> str:
+    """Normalize XML for comparison."""
+    normalized = minidom.parseString(xml_text).toprettyxml(indent=" ")
+    # Remove blank lines
+    return "\n".join([line for line in normalized.splitlines() if line.strip()])
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_path", help="Path to the model in JSON or XML", required=True
+    )
+    args = parser.parse_args()
+
+    model_path = pathlib.Path(args.model_path)
+
+    if not model_path.exists():
+        print(f"--model_path does not exist: {model_path}", file=sys.stderr)
+        sys.exit(1)
+
+    text = model_path.read_text(encoding="utf-8")
+    stripped_text = text.strip()
+
+    if len(stripped_text) == 0:
+        print(f"Model {model_path} is empty.", file=sys.stderr)
+        sys.exit(1)
+
+    # NOTE (mristin):
+    # We assume by convention that the main object in the integration test is defined as
+    # the class ``Root``.
+    root: types.Root
+
+    first_character = stripped_text[0]
+
+    jsonable = None  # type: Optional[Any]
+
+    source_format: Literal["json", "xml"]
+
+    if first_character == "{":
+        try:
+            jsonable = json.loads(text)
+        except json.JSONDecodeError as exception:
+            print(
+                f"Failed to parse JSON from {model_path}: {exception}", file=sys.stderr
+            )
+            sys.exit(1)
+
+        try:
+            root = jsonization.root_from_jsonable(jsonable)
+        except jsonization.DeserializationException as exception:
+            print(f"Failed to parse {model_path}: {exception}", file=sys.stderr)
+            sys.exit(1)
+
+        source_format = "json"
+
+    elif first_character == "<":
+        try:
+            root = xmlization.root_from_str(text)
+        except xmlization.DeserializationException as exception:
+            print(f"Failed to parse {model_path}: {exception}", file=sys.stderr)
+            sys.exit(1)
+
+        source_format = "xml"
+
+    else:
+        print(
+            f"Unrecognized first non-whitespace character in {model_path}, "
+            f"so we do not know how to parse the file: {first_character!r}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    for error in verification.verify(root):
+        print(
+            f"The model from {model_path} failed to pass verification "
+            f"at {error.path}: {error.cause}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    if source_format == "json":
+        another_jsonable = jsonization.to_jsonable(root)
+
+        assert jsonable is not None
+
+        original_pretty = json.dumps(jsonable, indent=2, ensure_ascii=False)
+        another_pretty = json.dumps(another_jsonable, indent=2, ensure_ascii=False)
+
+        if original_pretty != another_pretty:
+            diff = "\n".join(
+                difflib.unified_diff(
+                    original_pretty.splitlines(),
+                    another_pretty.splitlines(),
+                    fromfile="original",
+                    tofile="roundtrip",
+                    lineterm="",
+                )
+            )
+            print(
+                f"JSON round-trip mismatch for {model_path}:\n{diff}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+    elif source_format == "xml":
+        assert isinstance(root, types.Root)
+
+        another_text = xmlization.to_str(root)
+
+        if normalize_xml(text) != normalize_xml(another_text):
+            diff = "\n".join(
+                difflib.unified_diff(
+                    normalize_xml(text).splitlines(),
+                    normalize_xml(another_text).splitlines(),
+                    fromfile="original",
+                    tofile="roundtrip",
+                    lineterm="",
+                )
+            )
+            print(
+                f"XML round-trip mismatch for {model_path}:\n{diff}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+    else:
+        assert_never(source_format)
+
+
+if __name__ == "__main__":
+    main()