development-framework-cli 1.7.3__py3-none-any.whl

dm_cli/import_entity.py

@@ -0,0 +1,128 @@
+import io
+import json
+from json import JSONDecodeError
+from pathlib import Path
+from zipfile import ZipFile
+
+from requests import Response
+from rich import print
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    retry_if_not_exception_type,
+    retry_if_not_result,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+
+from .dmss import ApplicationException, dmss_api
+from .dmss_api.exceptions import ApiException, NotFoundException, ServiceException
+from .import_package import import_package_tree
+from .package_tree_from_zip import package_tree_from_zip
+from .state import state
+from .utils.reference import replace_relative_references
+from .utils.utils import (
+    concat_dependencies,
+    console,
+    destination_is_root,
+    ensure_package_structure,
+)
+from .utils.zip import zip_all
+
+
+@retry(
+    wait=wait_random_exponential(multiplier=1, max=60),
+    stop=stop_after_attempt(5),
+    reraise=True,
+    retry=retry_if_exception_type(ServiceException),
+)
+def import_document(source_path: Path, destination: str, document: dict):
+    remote_dependencies = dmss_api.export_meta(destination)
+    old_dependencies = {dependency["alias"]: dependency for dependency in remote_dependencies.get("dependencies", [])}
+
+    dependencies = concat_dependencies(
+        new_dependencies=document.get("_meta_", {}).get("dependencies", []),
+        old_dependencies=old_dependencies,
+        filename=source_path.name,
+    )
+
+    # Replace references
+    prepared_document = replace_relative_references(document, dependencies, destination)
+
+    document_json_str = json.dumps(prepared_document)
+    dmss_api.document_add(
+        destination,
+        document_json_str,
+        files=[],
+    )
+
+
+def import_single_entity(source_path: Path, destination: str, validate: bool = False):
+    ensure_package_structure(Path(destination))
+    print(f"Importing ENTITY '{source_path.name}' --> '{destination}'")
+
+    try:  # Load the JSON document
+        with open(source_path, "r") as fh:
+            if Path(source_path).suffix == ".json":
+                content = json.load(fh)
+                if validate:
+                    print(f"Validating {source_path}", end="")
+                    dmss_api.validate_entity(content)
+                    print(" [green]✓[/green]")
+                import_document(source_path, destination, content)
+            else:
+                print(f"Unsupported file type {source_path}")
+    except JSONDecodeError:
+        raise Exception(f"Failed to load the file '{source_path.name}' as a JSON document")
+
+
+def remove_by_path_ignore_404(target: str):
+    try:
+        dmss_api.document_remove(target)
+    except NotFoundException:
+        pass
+
+
+@retry(
+    wait=wait_random_exponential(multiplier=1, max=60),
+    stop=stop_after_attempt(5),
+    reraise=True,
+    retry=retry_if_exception_type(ServiceException),
+)
+def import_folder_entity(
+    source_path: Path,
+    destination: str,
+    raw_package_import: bool = False,
+    resolve_local_ids: bool = False,
+) -> dict:
+    destination_path = Path(destination)
+
+    # Check if target already exists on remote. Then delete or raise exception
+    target = f"{destination}/{source_path.name}"
+    exists = dmss_api.document_check(target)
+    if exists:
+        if not state.force:
+            raise ValueError(f"Failed to upload to '{target}' - It already exists.")
+        console.print(f"'{target}' already exists.  Replacing it...", style="dark_orange")
+        dmss_api.document_remove(target)
+
+    dependencies = {}
+    is_root = destination_is_root(destination_path)
+    if not is_root:
+        ensure_package_structure(destination_path)
+        remote_dependencies = dmss_api.export_meta(f"{destination}")
+        dependencies = {dependency["alias"]: dependency for dependency in remote_dependencies.get("dependencies", [])}
+
+    memory_file = io.BytesIO()
+    with ZipFile(memory_file, mode="w") as zip_file:
+        zip_all(
+            zip_file,
+            source_path,
+            write_folder=True,
+        )
+    memory_file.seek(0)
+
+    package = package_tree_from_zip(
+        destination, memory_file, is_root=is_root, extra_dependencies=dependencies, source_path=source_path
+    )
+    import_package_tree(package, destination, raw_package_import, resolve_local_ids)

dm_cli/import_package.py

@@ -0,0 +1,173 @@
+import io
+import json
+from json import JSONDecodeError
+from pathlib import Path
+from typing import Dict, List
+from uuid import uuid4
+
+from rich.console import Console
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    retry_if_not_exception_type,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+from tqdm import tqdm
+
+from .dmss import ApplicationException, dmss_api
+from .dmss_api.exceptions import ServiceException
+from .domain import Dependency, File, Package
+from .utils.reference import replace_relative_references
+from .utils.resolve_local_ids import resolve_local_ids_in_document
+from .utils.utils import concat_dependencies, replace_global_addresses
+
+console = Console()
+
+
+def add_object_to_package(path: Path, package: Package, object: io.BytesIO) -> None:
+    if len(path.parts) == 1:  # End of path means the actual document
+        file = File(
+            uid=str(uuid4()),  # This UID will be the data source ID for this file
+            name=object.name,
+            content=object,
+            path=path,
+        )
+        package.content.append(file)
+        return
+    items = [item for item in package.content if not isinstance(item, File)]
+    sub_folder = next((p for p in items if p["name"] == path.parts[0]), None)
+    if not sub_folder:  # If the sub folder has not already been created on parent, create it
+        sub_folder = Package(name=path.parts[0], parent=package)
+        package.content.append(sub_folder)
+
+    new_path = str(path).split("/", 1)[1]  # Remove first element in path before stepping down
+    return add_object_to_package(Path(new_path), sub_folder, object)
+
+
+def add_file_to_package(path: Path, package: Package, document: dict) -> None:
+    if len(path.parts) == 1:  # End of path means the actual document
+        if path.name.endswith("package.json"):
+            # if document is a package.json file, add meta info to package instead of adding it to content list.
+            package.meta = document.get("_meta_", {})
+            return
+        # Create a UUID if the document does not have one
+        package.content.append({**document, "_id": document.get("_id", str(uuid4()))})
+        return
+    items = [item for item in package.content if not isinstance(item, File)]
+    sub_folder = next((p for p in items if p["name"] == path.parts[0]), None)
+    if not sub_folder:  # If the sub folder has not already been created on parent, create it
+        sub_folder = Package(name=path.parts[0], parent=package)
+        package.content.append(sub_folder)
+
+    new_path = str(path).split("/", 1)[1]  # Remove first element in path before stepping down
+    return add_file_to_package(Path(new_path), sub_folder, document)
+
+
+def add_package_to_package(path: Path, package: Package) -> None:
+    if len(path.parts) == 1:
+        package.content.append(Package(name=path.parts[0], parent=package))
+        return
+    items = [item for item in package.content if not isinstance(item, File)]
+    sub_folder = next((p for p in items if p["name"] == path.parts[0]), None)
+    if not sub_folder:  # If the sub folder has not already been created on parent, create it
+        sub_folder = Package(name=path.parts[0], parent=package)
+        package.content.append(sub_folder)
+
+    new_path = str(path).split("/", 1)[1]  # Remove first element in path before stepping down
+    return add_package_to_package(Path(new_path), sub_folder)
+
+
+def import_package_tree(package: Package, destination: str, raw_package_import: bool, resolve_local_ids: bool) -> None:
+    destination_parts = destination.split("/")
+    data_source = destination_parts[0]
+
+    if raw_package_import:
+        dmss_api.document_add_simple(data_source, body=package.to_dict())
+    else:
+        dmss_api.document_add(
+            destination,
+            json.dumps(package.to_dict()),
+            files=[],
+        )
+
+    import_package_content(package, data_source, destination, resolve_local_ids)
+
+
+@retry(
+    wait=wait_random_exponential(multiplier=1, max=60),
+    stop=stop_after_attempt(5),
+    reraise=True,
+    retry=retry_if_exception_type(ServiceException),
+)
+def import_package_content(package: Package, data_source: str, destination: str, resolve_local_ids: bool) -> None:
+    files: List[File] = []
+    entities: List[dict] = []
+    package.traverse_documents(
+        lambda document, **kwargs: files.append(document) if isinstance(document, File) else entities.append(document)
+    )
+    uploaded_file_ids = {}
+    if len(files) > 0:
+        with tqdm(files, desc=f"  Adding files") as bar:
+            for file in files:
+                dmss_api.file_upload(data_source, json.dumps({"file_id": file.uid}), file.content)
+                uploaded_file_ids[f"dmss:/{file.content.destination}/{file.path.stem}"] = file.uid
+                bar.update()
+
+    def upload_global_file(address: str) -> str:
+        """Handling uploading of global files."""
+        filepath = Path(address)
+        if not filepath.is_file():
+            raise ApplicationException(
+                f"Tried to upload file with address '{address}'. The file was not found", data=package.to_dict()
+            )
+        if filepath.suffix != ".json":
+            # Binary files
+            with open(address, "rb") as f:
+                file_like = io.BytesIO(f.read())
+            file_like.name = filepath.stem
+            global_id = str(uuid4())
+            dmss_api.blob_upload(data_source, global_id, file_like)
+            return global_id
+        else:
+            try:
+                with open(address) as f:
+                    global_document = json.load(f)
+                # Get dependencies from package
+                dependencies: Dict[str, Dependency] = {
+                    dependency["alias"]: Dependency(**dependency)
+                    for dependency in package.meta.get("dependencies", [])
+                }
+                # Add dependencies from entity
+                dependencies = concat_dependencies(
+                    global_document.get("_meta_", {}).get("dependencies", []), dependencies, address
+                )
+                global_document = replace_relative_references(
+                    global_document,
+                    dependencies,
+                    destination,
+                    file_path=address,
+                )
+                global_id = dmss_api.document_add_simple(data_source, global_document)
+                return global_id
+            except JSONDecodeError:
+                raise Exception(f"Failed to load the file '{address}' as a JSON document")
+
+    if len(entities) > 0:
+        with tqdm(entities, desc=f"  Adding entities") as bar:
+            for entity in entities:
+                document = replace_global_addresses(entity, destination, uploaded_file_ids, upload_global_file)
+                if resolve_local_ids:
+                    name = f"/{document.get('name')}" if document.get("name") else f" of type {document.get('type')}"
+                    document = resolve_local_ids_in_document(document)
+                    print(f"Successfully resolved local IDs in:\t{destination}{name}")
+                dmss_api.document_add_simple(data_source, document)
+                bar.update()
+
+    packages: List[Package] = []
+    package.traverse_package(lambda package: packages.append(package))
+    if len(packages) > 0:
+        with tqdm(packages, desc=f"  Adding packages") as bar:
+            for package in packages:
+                dmss_api.document_add_simple(data_source, package.to_dict())
+                bar.update()

dm_cli/package_tree_from_zip.py

@@ -0,0 +1,120 @@
+import io
+import json
+from json import JSONDecodeError
+from pathlib import Path
+from typing import Dict, Union
+from zipfile import ZipFile
+
+from .domain import Dependency, File, Package
+from .import_package import (
+    add_file_to_package,
+    add_object_to_package,
+    add_package_to_package,
+)
+from .utils.reference import replace_relative_references
+from .utils.utils import concat_dependencies
+
+
+def package_tree_from_zip(
+    destination: str,
+    zip_package: io.BytesIO,
+    is_root: bool = True,
+    extra_dependencies: Union[Dict[str, Dependency], None] = None,
+    source_path: Path = None,
+) -> Package:
+    """
+    Converts a Zip-folder into a DMSS Package structure.
+
+    Inserting UUID4's between any references, converting relative paths to absolute paths,
+    and dependencyAliases to absolute addresses.
+
+    @param destination: A string with the documentId for the target. Only a data source is allowed
+    @param zip_package: A zip-folder represented as an in-memory io.BytesIO object
+    @param source_path: path to the root folder
+
+    @return: A Package object with sub folders(Package) and documents(dict)
+    """
+
+    with ZipFile(zip_package) as zip_file:
+        folder_name = zip_file.filelist[0].filename.split("/", 1)[0]
+
+        # Find the root packages package.json file
+        package_file = next(
+            (z for z in zip_file.filelist if z.filename == f"{folder_name}/package.json"),
+            None,
+        )
+
+        package_entity = json.loads(zip_file.read(package_file.filename)) if package_file else {}
+        if package_file in zip_file.filelist:
+            zip_file.filelist.remove(package_file)
+        dependencies: Dict[str, Dependency] = {
+            dependency["alias"]: Dependency(**dependency)
+            for dependency in package_entity.get("_meta_", {}).get("dependencies", [])
+        }
+        if extra_dependencies:
+            dependencies.update(extra_dependencies)
+        root_package = Package(
+            name=package_entity.get("name", folder_name),
+            is_root=is_root,
+            meta=package_entity.get("_meta_"),
+        )
+        # Construct a nested Package object of the package to import
+        for file_info in zip_file.filelist:
+            filename = file_info.filename.split("/", 1)[1]  # Remove RootPackage prefix
+            if file_info.is_dir():
+                if filename == "":  # Skip rootPackage
+                    continue
+                add_package_to_package(Path(filename), root_package)
+                continue
+            if Path(filename).suffix != ".json":
+                file_like = io.BytesIO(zip_file.read(f"{folder_name}/{filename}"))
+                file_like.name = Path(filename).name  # stem
+                file_like.destination = Path(f"/{destination}/{folder_name}/{filename}").parent
+                add_object_to_package(Path(filename), root_package, file_like)
+                continue
+            try:
+                json_doc = json.loads(zip_file.read(f"{folder_name}/{filename}"))
+            except JSONDecodeError:
+                raise Exception(f"Failed to load the file '{filename}' as a JSON document")
+
+            add_file_to_package(Path(filename), root_package, json_doc)
+
+            # Add dependencies from entity to the global dependencies list
+            dependencies = concat_dependencies(
+                json_doc.get("_meta_", {}).get("dependencies", []), dependencies, filename
+            )
+
+        def replace(document, file_path):
+            if not isinstance(document, File):
+                document = replace_relative_references(
+                    document,
+                    dependencies,
+                    destination,
+                    file_path=file_path,
+                    source_path=source_path,
+                )
+            return document
+
+        # Now that we have the entire package as a Package tree, traverse it, and replace relative references
+        root_package.traverse_documents(
+            lambda document, file_path: replace(document, file_path),
+            update=True,
+        )
+        root_package.meta = replace_relative_references(
+            root_package.meta,
+            dependencies,
+            destination,
+            file_path=root_package.path(),
+            source_path=source_path,
+        )
+        root_package.traverse_package(
+            lambda package: replace_relative_references(
+                package.meta,
+                dependencies,
+                destination,
+                file_path=package.path(),
+                source_path=source_path,
+            )
+        )
+
+    return root_package

dm_cli/state.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class State:
+    force: bool = False
+    token: str = "no-token"
+    dmss_url: str = "http://localhost:5000"
+    debug: bool = False
+
+
+state = State()

dm_cli/utils/file_structure.py

@@ -0,0 +1,31 @@
+import os
+from pathlib import Path
+from typing import List
+
+import typer
+
+DATA_SOURCE_DEF_FILE_EXT: str = "json"
+
+
+def get_app_dir_structure(path: Path) -> [Path, Path]:
+    app_dir = Path(path)
+    if not app_dir.is_dir():
+        raise FileNotFoundError(f"The path '{path}' is not a directory.")
+
+    # Check for presence of expected directories, 'data_sources' and 'data'
+    data_sources_dir = path.joinpath("data_sources")
+    data_dir = path.joinpath("data")
+    if not data_sources_dir.is_dir() or not data_dir.is_dir():
+        print(f"The directory '{path.name}' does not have the expected structure. It should contain two directories;")
+        print("""
+            ├── data
+            └── data_sources
+        """)
+        raise typer.Exit(code=1)
+
+    return data_sources_dir, data_dir
+
+
+def get_json_files_in_dir(path: Path) -> List[str]:
+    """Get all JSON files found in <path>."""
+    return [filename for filename in os.listdir(path) if filename.endswith(f".{DATA_SOURCE_DEF_FILE_EXT}")]

dm_cli/utils/reference.py

@@ -0,0 +1,129 @@
+from os.path import normpath
+from pathlib import Path
+from typing import Dict, List, Union
+
+from ..dmss import ApplicationException
+from ..domain import Dependency
+from ..enums import SIMOS, BuiltinDataTypes, ReferenceTypes
+from .utils import resolve_dependency
+
+
+def resolve_reference(
+    reference: str, dependencies: Dict[str, Dependency], destination: str, file_path: str | None
+) -> str:
+    destination = destination.rstrip("/")
+    root_package = file_path.split("/", 1)[0] if file_path else ""
+    try:
+        if "://" in reference or reference == "_default_" or reference[0] == "^" or reference[0] == "~":
+            return reference
+        if ":" in reference:
+            return resolve_dependency(reference, dependencies)
+        if reference[0] == ".":
+            normalized_dotted_ref: str = normpath(f"{file_path}/{reference}")
+            return f"dmss://{destination}/{normalized_dotted_ref}"
+        if reference[0] == "/":
+            data_source = destination.split("/")[0]
+            return f"dmss://{data_source}{reference}"
+        return f"dmss://{destination}/{root_package}/{reference}"
+    except ApplicationException as ex:
+        ex.debug = f"File: {file_path}, Destination: {destination}, Root: {root_package}"
+        raise ex
+
+
+def replace_relative_references(
+    value: dict | list,
+    dependencies: Dict[str, Dependency],
+    destination: str,
+    file_path: Union[str, None] = None,
+    source_path: Path = None,
+) -> Union[str, List[str], dict]:
+    """
+    Takes a dict or list, and returns the passed value, with
+    relative references replaced with absolute ones.
+
+    For Blob-entities; insert the binary data from the file into the entity.
+    It digs down on complex types
+
+    @param value: Dict or list of an entity
+    @param dependencies: A dict containing the dependencies of the document
+    @param destination: The name of the data source where the document should be stored
+    @param file_path: The path to the directory containing the documents
+
+    When importing packages, the following additional parameter is required:
+    @param zip_file:
+    @param source_path: path to the root folder
+    """
+
+    def _resolve_reference(inner_value: str | None):
+        if not inner_value:
+            return inner_value
+        return resolve_reference(
+            inner_value,
+            dependencies,
+            destination,
+            file_path,
+        )
+
+    def _replace_relative(inner_value: dict | list):
+        return replace_relative_references(inner_value, dependencies, destination, file_path, source_path)
+
+    if isinstance(value, dict):
+        if not value:
+            return value
+
+        if not value.get("type"):
+            raise KeyError(f"Object is missing the required 'type' attribute. File: '{file_path}'")
+
+        value["type"] = _resolve_reference(value["type"])
+
+        match _resolve_reference(value["type"]):
+            case SIMOS.REFERENCE.value:
+                if value["referenceType"] == ReferenceTypes.LINK.value:
+                    value["address"] = _resolve_reference(value["address"])
+                else:
+                    # Handle storage references
+                    local_file_path = "/".join(str(source_path).split("/")[:-1])
+                    if value["address"][0] == ".":
+                        raise ApplicationException(
+                            f"Relative references by . are not supported", data=value, debug=file_path
+                        )
+                    value["address"] = f"{local_file_path}{value['address']}"
+                return value
+
+            case SIMOS.ATTRIBUTE.value:
+                if enum_type := value.get("enumType"):
+                    value["enumType"] = _resolve_reference(enum_type)
+                if value["attributeType"] not in [data_type.value for data_type in BuiltinDataTypes]:
+                    value["attributeType"] = _resolve_reference(value["attributeType"])
+                    if default := value.get("default"):
+                        value["default"] = _replace_relative(default)
+                return value
+
+            case SIMOS.BLUEPRINT.value:
+                value["extends"] = [_resolve_reference(ext_from) for ext_from in value.get("extends", [])]
+                value["attributes"] = [_replace_relative(attr) for attr in value.get("attributes", [])]
+                if meta := value.get("_meta_"):
+                    value["_meta_"] = _replace_relative(meta)
+                return value
+
+            case SIMOS.RECIPE_LINK.value:
+                value["_blueprintPath_"] = _resolve_reference(value["_blueprintPath_"])
+                if initial_recipe := value.get("initialUiRecipe"):
+                    value["initialUiRecipe"] = _replace_relative(initial_recipe)
+                if uiRecipes := value.get("uiRecipes"):
+                    value["uiRecipes"] = _replace_relative(uiRecipes)
+                return value
+
+            case _:  # The value is a dict, but of unknown type. Need to dig through it recursively
+                for key, inner_value in value.items():
+                    if isinstance(inner_value, dict) or isinstance(inner_value, list):
+                        value[key] = _replace_relative(inner_value)
+                return value
+
+    if isinstance(value, list):
+        if value and (isinstance(value[0], dict) or isinstance(value[0], list)):
+            return [_replace_relative(v) for v in value]
+        # It's an empty or primitive list. Dig no further
+        return value
+
+    raise ValueError(f"Function can only be called on dicts and lists. Got {value}")