development-framework-cli 1.7.3__py3-none-any.whl

dm_cli/utils/resolve_local_ids.py

@@ -0,0 +1,142 @@
+import re
+
+
+def search_in_list(
+    document: list, pattern: str, target_attr: str, path: str, parent_was_dict: bool, targets: dict, references: dict
+):
+    attribute_types = map(lambda attr: (attr, type(attr)), document)
+    for i, at in enumerate(attribute_types):
+        a = at[0]
+        t = at[1]
+        if is_primitive(t):
+            result = search_in_primitive(a, pattern)
+            if result:
+                references[f"{path}[{i}]"] = result
+        else:
+            action = type_to_action[t]
+            action(a, pattern, target_attr, f"{path}[{i}]", False, targets, references)
+
+
+def search_in_primitive(val, pattern: str):
+    match = re.match(pattern, str(val))
+    if match:
+        return match.group(1)
+
+
+def is_primitive(t: type):
+    return t == str or t == int or t == float or t == bool
+
+
+def is_list(t: type):
+    return t == list
+
+
+def is_dict(t: type):
+    return t == dict
+
+
+def search_in_dict(
+    document: dict, pattern: str, target_attr: str, path: str, parent_was_dict: bool, targets: dict, references: dict
+):
+    """Search for internal references in 'document', referencing to a target attribute.
+
+    Uses the methods 'search_in_list' and 'search_in_primitive' to continue searching in attributes of these types
+
+    Args:
+        document: the input document which contains internal references
+        pattern: (what does a 'reference' look like?) regex which tries to match the reference structure, defaults to
+            ^.$<some_id>. Is used to populate the lookup_of_references table
+        target_attr: (where is the reference pointing to?) the attribute which contains the id which is being referred
+            to is used to populate the lookup_of_targets table
+        path: the absolute path at each level in the search
+        parent_was_dict: flag to track the type of the parent
+        targets: dict to track discovered attributes that define local ids
+        references: dict to track discovered references to local ids
+
+    Returns:
+        Two lookup tables: 'references' dict, which mapping from the location of the reference (path), to the
+        local id, and 'targets' dict, which is a mapping from the local id to the location of the element which owns
+        that id (absolute local path)
+    """
+
+    attribute_types = map(lambda attr: (attr, type(document[attr])), list(document.keys()))
+    for a, t in attribute_types:
+        val = document[a]
+        if is_primitive(t):
+            if a == target_attr:
+                parent_path = path.rsplit("[", 1)[0]
+                final_path = path if parent_was_dict else f"{parent_path}({target_attr}={val})"
+                targets[val] = final_path
+            result = search_in_primitive(val, pattern)
+            if result:
+                references[f"{path}.{a}"] = result
+        else:
+            action = type_to_action[t]
+            action(document[a], pattern, target_attr, f"{path}.{a}", True, targets, references)
+    return targets, references
+
+
+def _dig_and_replace(document, path_iterator: iter, replace: str):
+    attr = next(path_iterator, None)
+    if not attr:
+        return True
+    attr = int(attr) if is_list(type(document)) else attr
+    last_attribute = _dig_and_replace(document[attr], path_iterator, replace)
+    if last_attribute:
+        document[attr] = replace
+
+
+def dig_and_replace(targets: dict, references: dict, document):
+    """Transform references in 'references' to the absolute (local) path found in 'targets'
+
+    Traverses the document based on the path found in 'references' lookup table, and updates the entry with the
+    corresponding value found in the 'targets' lookup table (the local path to the target attribute)
+
+    Args:
+        references: mapping from the location of the reference (path), to the local id
+        targets: mapping from the local id to the location of the element which owns that id
+        document: dictionary document which contains references we want to transform/replace
+
+    Returns:
+        The final document with all the local id references resolved into local path references
+
+    """
+    for path, ref in references.items():
+        path_items = re.split("[\.\[\]]+", path)[1:]  # splits path on the '.', '[' and ']' characters
+        path_iterator = iter(path_items)
+        _dig_and_replace(document, path_iterator, targets[ref])
+    return document
+
+
+def resolve_local_ids_in_document(
+    document: dict,
+    pattern: str = r"\^\.\$(\w+)",
+    target_attr: str = "_id",
+    path: str = "^",
+    parent_was_dict: bool = True,
+):
+    """Transform local_id references in 'document' to absolute (local) paths
+
+    e.g.: customers[0].car.address points to cars[0][0]
+
+    (local id reference)         (absolute path reference)
+    "address": ^.$12345    ->    "address": ^.cars[0](_id=12345)
+
+    Args:
+        document: the input document which contains internal references
+        pattern: (what does a 'reference' look like?) regex which tries to match the reference structure, defaults to
+            ^.$<some_id>. Is used to populate the lookup_of_references table
+        target_attr: (where is the reference pointing to?) the attribute which contains the id which is being referred
+            to is used to populate the lookup_of_targets table
+        path: the absolute path at each level in the search
+        parent_was_dict: flag to track the type of the parent
+
+    Returns:
+        The final document with all the local id references resolved into local path references
+    """
+
+    targets, references = search_in_dict(document, pattern, target_attr, path, parent_was_dict, {}, {})
+    return dig_and_replace(targets, references, document)
+
+
+type_to_action = {list: search_in_list, dict: search_in_dict}

dm_cli/utils/utils.py

@@ -0,0 +1,161 @@
+import json
+import os
+import pprint
+from pathlib import Path
+from typing import Callable, Dict, List
+
+import typer
+from rich import print
+from rich.console import Console
+
+from dm_cli.dmss import ApplicationException, dmss_api
+from dm_cli.dmss_api.exceptions import ApiException, NotFoundException
+from dm_cli.utils.file_structure import get_app_dir_structure, get_json_files_in_dir
+
+from ..domain import Dependency, Package
+from ..enums import SIMOS
+
+console = Console()
+
+
+def resolve_dependency(type_ref: str, dependencies: Dict[str, Dependency]) -> str:
+    """Takes a type reference and dependencies. Returns the address for
+    the Blueprint, prefixed with which protocol should be used to fetch it.
+    Expected format is ALIAS:ADDRESS"""
+    tag, path = type_ref.split(":", 1)
+    path = path.strip(" /")
+    dependency = dependencies.get(tag)
+    if not dependency:
+        raise ApplicationException(f"No dependency with alias '{tag}' was found in the entities dependencies list")
+    address = dependency.address.strip(" /")
+
+    if dependency.protocol == "dmss":
+        return f"dmss://{address}/{path}"
+    if dependency.protocol == "http":
+        return f"http://{address}/{path}"
+
+    raise ApplicationException(f"Protocol '{dependency.protocol}' is not a valid protocol for resolving dependencies")
+
+
+def concat_dependencies(
+    new_dependencies: List[dict], old_dependencies: Dict[str, Dependency], filename: str
+) -> Dict[str, Dependency]:
+    entity_dependencies = {dependency["alias"]: Dependency(**dependency) for dependency in new_dependencies}
+    alias_intersect = entity_dependencies.keys() & old_dependencies.keys()
+
+    # If there are duplicated aliases, raise error if they are not identical to the existing one
+    for duplicated_alias in alias_intersect:
+        if entity_dependencies[duplicated_alias] != old_dependencies[duplicated_alias]:
+            raise ApplicationException(f"Conflicting dependency alias(es) in file '{filename}'. '{alias_intersect}'")
+    old_dependencies.update(entity_dependencies)
+    return old_dependencies
+
+
+def document_already_exists(api_exception: ApiException) -> bool:
+    error = json.loads(api_exception.body)
+    if error.get("type") == "BadRequestException" and "already exists" in error.get("message"):
+        print(f"ERROR: {error.get('message')}")
+        return True
+    return False
+
+
+def add_package_to_path(name: str, path: Path):
+    package = Package(name, is_root=len(path.parts) == 2)
+    dmss_api.document_add(
+        str(path.parent),
+        json.dumps(package.to_dict()),
+        files=[],
+    )
+
+
+def replace_global_addresses(
+    document: dict, data_source_id: str, files_to_upload: dict, upload_global_file: Callable
+) -> dict:
+    # This is for contained Files(blob data)? If so they should be embedded in the json document, not uploaded as a blob
+    if document["type"] == SIMOS.REFERENCE.value and document["address"] in files_to_upload:
+        blob_id = files_to_upload[document["address"]]
+        document["address"] = f"${blob_id}"
+    # This is for model contained, storage-uncontained documents. They are stored on disk in the "global" folder
+    if document["type"] == SIMOS.REFERENCE.value and document["referenceType"] == "storage":
+        global_id = upload_global_file(document["address"])
+        document["address"] = f"${global_id}"
+    for key, value in document.items():
+        if key == "_meta_":  # meta data can never contain blob data.
+            return document
+        if isinstance(value, dict) and value:
+            document[key] = replace_global_addresses(value, data_source_id, files_to_upload, upload_global_file)
+        if isinstance(value, list) and value:
+            if len(value) > 0 and isinstance(value[0], dict):
+                document[key] = [
+                    replace_global_addresses(item, data_source_id, files_to_upload, upload_global_file)
+                    for item in document[key]
+                ]
+    return document
+
+
+def destination_is_root(destination: Path) -> bool:
+    if len(destination.parts) > 1:
+        return False
+    return True
+
+
+def ensure_package_structure(path: Path):
+    """Create any missing packages in the provided path (mkdir -R)"""
+    try:
+        dmss_api.document_get(f"dmss://{path}/")
+    except NotFoundException as e:
+        error = json.loads(e.body)
+        if error["status"] != 404:
+            print(f"Target package '{path}' is likely corrupt!")
+            pprint.pformat(error)
+            raise typer.Exit(code=1)
+
+        if len(path.parts) > 2:  # We're at root package level. Do not check for datasource.
+            ensure_package_structure(path.parent)
+
+        add_package_to_path(path.name, path)
+        print(f"Target folder '{path.name}' was missing in '{path.parent}'. Created: ✓")
+
+
+def get_root_packages_in_data_sources(path: str) -> dict:
+    """
+    Generate a dict that contains what root packages are included in a data source. Example structure:
+    {
+        "dataSourceA": [rootPackageA, rootPackageB],
+        "dataSourceB": [rootPackageC]
+    }
+    """
+    data_sources_dir, data_dir = get_app_dir_structure(Path(path))
+    data_source_definitions: list[str] = get_json_files_in_dir(data_sources_dir)
+    root_packages_in_data_sources = {}
+    for data_source_definition in data_source_definitions:
+        with open(os.path.join(data_sources_dir, data_source_definition)) as file:
+            data_source_document = json.load(file)
+            # Global folders should be uploaded directly to the data source
+            # and should not be included in the root packages of the data source.
+            global_folders = data_source_document.get("global_folders", [])
+            data_source = data_source_document["name"]
+            data_source_path = os.path.join(data_dir, data_source)
+            root_packages: list[str] = [
+                folder
+                for folder in os.listdir(data_source_path)
+                if os.path.isdir(os.path.join(data_source_path, folder)) and folder not in global_folders
+            ]
+            root_packages_in_data_sources[data_source] = root_packages
+    return root_packages_in_data_sources
+
+
+def validate_entities_in_data_sources(data_source_contents: dict):
+    """Run validation on entities in data sources.
+    data_source_contents is a dict that contains what root packages are included in a data source. Example structure:
+    {
+        "dataSourceA": [rootPackageA, rootPackageB],
+        "dataSourceB": [rootPackageC]
+    }
+
+    """
+    for data_source_name in data_source_contents:
+        root_packages = data_source_contents[data_source_name]
+        for root_package_name in root_packages:
+            print("Validating entities in: ", f"{os.path.join(data_source_name, root_package_name)}")
+            dmss_api.validate_existing_entity(f"{os.path.join(data_source_name, root_package_name)}")

dm_cli/utils/zip.py

@@ -0,0 +1,70 @@
+import os
+from pathlib import Path
+from zipfile import ZipFile
+
+import emoji
+
+from ..dmss import ApplicationException
+
+
+def zip_all(zip_file: ZipFile, path: str, real_name="", write_folder: bool = True):
+    basename = os.path.basename(path)
+    if os.path.isdir(path):
+        if real_name == "":
+            real_name = basename
+        if write_folder:
+            zip_file.write(path, os.path.join(real_name))
+        for root, dirs, files in os.walk(path):
+            for d in dirs:
+                zip_all(
+                    zip_file,
+                    os.path.join(root, d),
+                    os.path.join(real_name, d),
+                    write_folder=write_folder,
+                )
+            for f in files:
+                zip_file.write(os.path.join(root, f), os.path.join(real_name, f))
+            break
+    elif os.path.isfile(path):
+        zip_file.write(path, os.path.join(real_name, basename))
+    else:
+        pass
+
+
+def unpack_and_save_zipfile(export_location: str, zip_file: ZipFile):
+    """Unpack zipfile and save it to export_location. It is assumed that zip file only contains json files and folders.
+    If file or folder to export already exists, an exception is raised.
+    """
+    zip_file_unpacked_path = f"{export_location}/{zip_file.filename.rstrip('.zip')}"
+
+    zip_has_single_file_and_no_folders = (
+        len(zip_file.filelist) == 1 and zip_file.filelist[0].filename.split("/")[0] == ""
+    )
+    if zip_has_single_file_and_no_folders:
+        # If single file in the zip file (and the file is not inside a folder), the unpacked path has .json ending
+        # (we assume the zip file always contains json files)
+        zip_file_unpacked_path += ".json"
+
+    if Path(zip_file_unpacked_path).exists():
+        print(emoji.emojize(f"\t:error: File or folder '{zip_file_unpacked_path}' already exists. Exiting."))
+        raise ApplicationException("Path already exists")
+
+    zip_file.extractall(path=export_location)
+    print(f"Saved unpacked zip file to '{zip_file_unpacked_path}'.")
+
+
+def save_as_zip_file(export_location: str, filename: str, data: str):
+    """Save binary data into a zip file on the local disk.
+    If file or folder to export already exists, an exception is raised.
+    """
+    if not filename.endswith(".zip"):
+        raise ApplicationException(message="file ending .zip must be included in filename!")
+    saved_zip_file_path = f"{export_location}/{filename}"
+
+    if Path(saved_zip_file_path).exists():
+        print(emoji.emojize(f"\t:error: File or folder '{saved_zip_file_path}' already exists. Exiting."))
+        raise ApplicationException("Path already exists")
+
+    with open(saved_zip_file_path, "wb") as file:
+        file.write(data)
+        print(f"Wrote zip file to '{saved_zip_file_path}'")