splight-cli 4.9.0__tar.gz → 4.10.0__tar.gz

{splight_cli-4.9.0 → splight_cli-4.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: splight-cli
-Version: 4.9.0
+Version: 4.10.0
 Summary: Splight Command Line Interface
 Author: Splight Dev
 Author-email: dev@splight-ae.com
@@ -15,8 +15,6 @@ Requires-Dist: case-converter (==1.1.0)
 Requires-Dist: click (==8.0.4)
 Requires-Dist: click-default-group (==1.2.2)
 Requires-Dist: colorama (==0.4.4)
-Requires-Dist: deepdiff (==6.3.1)
-Requires-Dist: mock (>=5.1.0,<6.0.0)
 Requires-Dist: packaging (==23.0)
 Requires-Dist: pathspec (==0.11.1)
 Requires-Dist: py7zr (==0.20.8)
@@ -24,6 +22,7 @@ Requires-Dist: requests (==2.31.0)
 Requires-Dist: rich (==13.0.1)
 Requires-Dist: setuptools (==60.9.3)
 Requires-Dist: splight-lib (>=5.2.2,<6.0.0)
+Requires-Dist: toposort (>=1.10,<2.0)
 Requires-Dist: typer (==0.9.0)
 Description-Content-Type: text/markdown
 

{splight_cli-4.9.0 → splight_cli-4.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "splight-cli"
-version = "4.9.0"
+version = "4.10.0"
 description = "Splight Command Line Interface"
 authors = ["Splight Dev <dev@splight-ae.com>"]
 readme = "README.md"
@@ -20,8 +20,7 @@ PyYAML = "6.0.1"
 rich = "13.0.1"
 pathspec = "0.11.1"
 packaging = "23.0"
-deepdiff = "6.3.1"
-mock = "^5.1.0"
+toposort = "^1.10"
 
 [tool.poetry.group.dev.dependencies]
 black = "23.3.0"
@@ -32,6 +31,7 @@ flake8 = "^6.1.0"
 pre-commit = "3.2.2"
 pytest = "^7.4.3"
 coverage = "^7.3.2"
+mock = "^5.1.0"
 
 [tool.poetry.scripts]
 splight = "splight_cli.cli:app"

splight_cli-4.10.0/splight_cli/solution/__init__.py

@@ -0,0 +1,88 @@
+import typer
+from rich.console import Console
+
+from splight_cli.context import check_credentials
+from splight_cli.solution.parser import Parser
+from splight_cli.solution.resource import ResourceManager
+from splight_cli.solution.state import State
+
+solution_app = typer.Typer(
+    name="Splight Solution",
+    add_completion=True,
+    rich_markup_mode="rich",
+    no_args_is_help=True,
+)
+
+console = Console()
+
+
+@solution_app.callback(invoke_without_command=True)
+def callback(ctx: typer.Context):
+    check_credentials(ctx)
+
+
+# FIXME: find a nice way to print the exceptions.
+# Take into account that most exception messages are short and do not have
+# that much content. I would include the traceback, or at least some part of it.
+# Another option is to not catch them, and show the entire error, so the user can
+# debug it.
+@solution_app.command()
+def apply(
+    _: typer.Context,
+    spec_file: str = typer.Argument(..., help="Path to the spec file."),
+    state_file: str = typer.Option(
+        "state.json", "--state", "-s", help="Path to the state file."
+    ),
+) -> None:
+    state = State(path=state_file)
+    state.load()
+
+    parser = Parser(spec_file=spec_file)
+    specs, dependency_graph = parser.parse()
+
+    manager = ResourceManager(
+        state=state,
+        specs=specs,
+        dependency_graph=dependency_graph,
+    )
+    manager.refresh()
+    plan = manager.plan()
+    if plan:
+        manager.apply()
+
+
+@solution_app.command()
+def plan(
+    _: typer.Context,
+    spec_file: str = typer.Argument(..., help="Path to the spec file."),
+    state_file: str = typer.Option(
+        "state.json", "--state", "-s", help="Path to the state file."
+    ),
+) -> None:
+    state = State(path=state_file)
+    state.load()
+
+    parser = Parser(spec_file=spec_file)
+    specs, dependency_graph = parser.parse()
+
+    manager = ResourceManager(
+        state=state,
+        specs=specs,
+        dependency_graph=dependency_graph,
+    )
+    manager.refresh()
+    manager.plan()
+
+
+@solution_app.command()
+def refresh(
+    _: typer.Context,
+    state_file: str = typer.Option(
+        "state.json", "--state", "-s", help="Path to the state file."
+    ),
+) -> None:
+    state = State(path=state_file)
+    state.load()
+
+    manager = ResourceManager(state=state)
+    manager.refresh()

splight_cli-4.10.0/splight_cli/solution/dict.py

@@ -0,0 +1,50 @@
+from typing import Any, Dict, List
+
+
+# FIXME: make this faster using a stack
+def walk_dict(data: Dict, current=[]):
+    result = []
+
+    if isinstance(data, Dict):
+        for key, value in data.items():
+            current.append(key)
+            result.extend(walk_dict(value, current.copy()))
+            current.pop()
+    elif isinstance(data, List):
+        for index, item in enumerate(data):
+            current.append(index)
+            result.extend(walk_dict(item, current.copy()))
+            current.pop()
+    else:
+        result.append((current.copy(), data))
+
+    return result
+
+
+def get_dict_value(path: List, data: Dict):
+    # Assumes the path exists
+    current = data
+    for key in path:
+        if isinstance(current, dict):
+            current = current[key]
+        elif isinstance(current, list):
+            current = current[int(key)]
+        else:
+            raise ValueError(f"Invalid path: {path}")
+    return current
+
+
+def set_dict_value(value: Any, path: List, data: Dict):
+    # Assumes the path exists
+    current = data
+    for key in path[:-1]:
+        current = current[key]
+
+    last_key = path[-1]
+
+    if isinstance(current, dict):
+        current[last_key] = value
+    elif isinstance(current, list):
+        current[int(last_key)] = value
+    else:
+        raise ValueError(f"Invalid path: {path}")

splight_cli-4.10.0/splight_cli/solution/parser.py

@@ -0,0 +1,103 @@
+import re
+from typing import Dict, List, Optional
+
+import yaml
+
+from splight_cli.solution.dict import walk_dict
+
+
+class DuplicateResourceError(Exception):
+    pass
+
+
+def parse_reference(value: str) -> Optional[Dict]:
+    """Return the reference if it matches the regex or None otherwise"""
+    # FIXME: make it so we are able to reference the 'name' of a resource.
+    # This is super easy to do:
+    # 1. Add '.id' and '.name' at the end of the regex
+    # 2. Save that accessor to the references dict in each spec.
+    # 3. Change the 'replace_references' function in ResourceManager so it
+    #    gets the correct attribute.
+
+    # Only process strings
+    if isinstance(value, str):
+        # Building this with format strings breaks the expression
+        pattern = r"^\${{(Asset|Attribute|Metadata|File|Function|Secret|Alert|Component|Routine)\.(\w+)}}$"
+
+        # Get the matches
+        match = re.match(pattern, value)
+
+        if match:
+            type, name = match.groups()
+            key = f"{type}:{name}"
+            return key
+
+
+class Parser:
+    def __init__(self, spec_file: str) -> None:
+        try:
+            with open(spec_file, "r") as fp:
+                self._spec_file_data = yaml.safe_load(fp)
+        except FileNotFoundError:
+            raise Exception(f"Spec file '{spec_file}' not found.")
+
+    def parse(self) -> List[Dict]:
+        # Here we build the dependency graph using the resource keys
+        dependency_graph = {}
+
+        # - Does not destroy all elements.
+        specs = {}
+        for resource_spec in self._spec_file_data or []:
+            name = resource_spec["name"]
+            type = resource_spec["type"]
+            arguments = resource_spec["arguments"]
+
+            # Parse each resource leaf value to see if its a reference
+            depends_on = set({})
+            references = []
+            for path, value in walk_dict(resource_spec["arguments"]):
+                result = parse_reference(value)
+                if result is not None:
+                    key = result
+
+                    # Must be done here, otherwise it would affect the dependency graph.
+                    # Paths and values are validated later on, since we may need data
+                    # that is available after creation.
+                    # This also prevents trying to delete a resource which has another
+                    # one referencing it.
+                    # Also prevents self-referencing.
+                    if key not in specs:
+                        raise ValueError(
+                            f"Reference '{value}' in resource '{name}' of type '{type}' points to a non existing resource."
+                        )
+
+                    reference = {
+                        "key": key,  # Key of the referenced resource
+                        "target": path,  # Where to put the value
+                        "string": value,  # The string value of the reference
+                    }
+
+                    references.append(reference)
+                    depends_on.add(key)
+
+            key = f"{type}:{name}"
+            if key in specs:
+                raise DuplicateResourceError(
+                    f"Resource '{name}' of type '{type}' is defined twice."
+                )
+
+            specs[key] = {
+                "name": name,
+                "type": type,
+                "depends_on": list(depends_on),
+                "references": references,
+                "arguments": arguments,
+            }
+
+            # Add its dependecies to the graph
+            dependency_graph[key] = depends_on
+
+        # NOTE: Before adding/changing features of the parser, insert a breakpoint
+        # here and inspect the specs and dependency_graph objects, to see the shape
+        # of the objects that the rest of the code needs.
+        return specs, dependency_graph

splight_cli-4.10.0/splight_cli/solution/resource/__init__.py

@@ -0,0 +1,5 @@
+from splight_cli.solution.resource.manager import ResourceManager
+
+__all__ = [
+    "ResourceManager",
+]

splight_cli-4.10.0/splight_cli/solution/resource/diff.py

@@ -0,0 +1,39 @@
+from typing import Dict
+
+from colorama import Fore, Style
+
+from splight_cli.solution.dict import get_dict_value, walk_dict
+
+
+def strike_text(text):
+    return "".join(["\u0336{}".format(c) for c in str(text)])
+
+
+def Diff(new_arguments: Dict, old_arguments: Dict) -> Dict:
+    # FIXME: this try catch mechanic could be simplified by improving
+    # the 'Dict' related functions.
+
+    # FIXME: Try replacing the QueryFilter in a function for a single value.
+    # The diff will show incorrectly.
+    diff = []
+    for path, new_value in walk_dict(new_arguments):
+        try:
+            get_dict_value(path, old_arguments)
+        except:
+            line = f"\t{Fore.GREEN}+{Style.RESET_ALL} {'.'.join([str(key) for key in path])}: {new_value}"
+            diff.append(line)
+            continue
+
+        old_value = get_dict_value(path, old_arguments)
+        if new_value != old_value:
+            line = f"\t{Fore.YELLOW}~{Style.RESET_ALL} {'.'.join([str(key) for key in path])}: {strike_text(old_value)} -> {new_value} "
+            diff.append(line)
+
+    for path, old_value in walk_dict(old_arguments):
+        try:
+            get_dict_value(path, new_arguments)
+        except:
+            line = f"\t{Fore.RED}-{Style.RESET_ALL} {'.'.join([str(key) for key in path])}: {strike_text(old_value)}"
+            diff.append(line)
+
+    return diff

splight_cli-4.10.0/splight_cli/solution/resource/logger.py

@@ -0,0 +1,36 @@
+from colorama import Style
+
+
+class ResourceLogger:
+    template = '{type}["{name}"]: {message} [id={id}]'
+
+    @staticmethod
+    def _bold(message: str) -> str:
+        return f"{Style.BRIGHT}{message}{Style.RESET_ALL}"
+
+    def resource(self, message: str, resource):
+        formatted_message = self.template.format(
+            type=resource.type,
+            name=resource.name,
+            id=resource.id,
+            message=message,
+        )
+        print(self._bold(formatted_message))
+
+    def event(
+        self,
+        message: str,
+        previous_line: bool = False,
+        new_line: bool = False,
+        bold: bool = False,
+    ):
+        formatted_message = ""
+        if previous_line:
+            formatted_message += "\n"
+        formatted_message += f"{self._bold(message)}" if bold else message
+        if new_line:
+            formatted_message += "\n"
+        print(formatted_message)
+
+    def diff(self, diff):
+        print("\n".join(diff))

splight_cli-4.10.0/splight_cli/solution/resource/manager.py

@@ -0,0 +1,237 @@
+from typing import Dict, List
+
+from toposort import toposort_flatten as toposort
+from typer import confirm
+
+from splight_cli.solution.resource.diff import Diff
+from splight_cli.solution.resource.logger import ResourceLogger
+from splight_cli.solution.resource.models import (
+    Asset,
+    Attribute,
+    File,
+    Function,
+    Metadata,
+    Resource,
+    Secret,
+)
+from splight_cli.solution.state import State
+
+type_map = {
+    "Asset": Asset,
+    "Attribute": Attribute,
+    "Metadata": Metadata,
+    "File": File,
+    "Function": Function,
+    "Secret": Secret,
+}
+
+
+# NOTE: One good thing about this approach, is that the code is idempotent.
+# That means, if it fails, you can just fix your spec issues (inspect the traceback)
+# and try again. The state won't loose its consistency.
+# FIXME: Paralelize the processing of non dependent resources.
+# Use toposort instead of toposort_flatten for this purpose.
+class ResourceManager:
+    def __init__(
+        self,
+        state: State,
+        specs: List[Resource] = [],
+        dependency_graph: Dict = {},
+    ) -> None:
+        self._state = state
+        self._specs = specs
+
+        # Sort the resource specs by dependency order
+        self._specs_order = toposort(dependency_graph)
+
+        self._logger = ResourceLogger()
+
+    def _create_resource(self, data: Dict):
+        type = data.get("type")
+        resource = type_map[type](**data)
+        return resource
+
+    def _update_references(
+        self, resource: Resource, resource_pool: List[Resource]
+    ):
+        # Replace each reference of this resource
+        for reference in resource.references:
+            reference_key = reference["key"]
+            reference_target = reference["target"]
+            reference_string = reference["string"]
+
+            value = resource_pool[reference_key].id
+            if value is None:
+                value = reference_string
+
+            # Replace it in our arguments.
+            resource.set_argument_value(reference_target, value)
+
+    def refresh(self):
+        for key in self._state.all():
+            data = self._state.get(key)
+            resource = self._create_resource(data)
+
+            self._logger.resource("Refreshing state...", resource)
+            resource.refresh()
+
+            self._state.update(
+                resource.key,
+                resource.dump(),
+            )
+            self._state.save()
+
+    def plan(self):
+        # I would not alter the flow of this function since it covers up
+        # all the possible edits you make to the spec file.
+
+        plan = {}
+        resources = {}
+
+        # FIXME: dependency_graph order in delete
+        # First remove the state items not present in the spec file
+        for key in self._state.all():
+            if key not in self._specs:
+                data = self._state.get(key)
+
+                resource = self._create_resource(data)
+
+                diff = Diff(new_arguments={}, old_arguments=resource.arguments)
+                plan[key] = {"operation": "deleted", "diff": diff}
+
+        # Iterate over the resource specs in the spec file
+        for key in self._specs_order:
+            # If this resource does not exist yet
+            if not self._state.contains(key):
+                data = self._specs[key]
+
+                resource = self._create_resource(data)
+                resources[key] = resource
+
+                # Replace each reference of this resource.
+                # If it does not exist, we set the reference text
+                # just to show in the plan what value will receive later.
+                self._update_references(resource, resources)
+
+                # We do not have previous arguments because this a new
+                # resource.
+                diff = Diff(new_arguments=resource.arguments, old_arguments={})
+                plan[key] = {"operation": "created", "diff": diff}
+
+            # Already exists in the engine
+            else:
+                state_data = self._state.get(key)
+                state_resource = self._create_resource(state_data)
+
+                spec_data = self._specs[key]
+                spec_resource = self._create_resource(spec_data)
+
+                # Replace each reference of this resource
+                # Same thing as before.
+                self._update_references(spec_resource, resources)
+
+                diff = Diff(
+                    new_arguments=spec_resource.arguments,
+                    old_arguments=state_resource.arguments,
+                )
+
+                resources[key] = state_resource
+
+                # This resource changed if the diff contains at least a line.
+                # We ignore this resource if the arguments stay the same.
+                # Note that we are comparing both arguments with their references
+                # replaced.
+                if diff:
+                    plan[key] = {"operation": "updated", "diff": diff}
+
+        if not plan:
+            self._logger.event(
+                "Your infrastructure matches the configuration."
+            )
+        else:
+            # Just print some messages and the diff for each resource
+            self._logger.event(
+                "Splight solution will perform the following actions:",
+            )
+            for key, data in plan.items():
+                action = data["operation"]
+                diff = data["diff"]
+
+                type, name = key.split(":")
+                self._logger.event(
+                    f"Resource '{name}' of type '{type}' will be {action}:",
+                    previous_line=True,
+                )
+                self._logger.diff(diff)
+
+        return plan
+
+    def apply(self):
+        # I would not alter the flow of this function since it covers up
+        # all the possible edits you make to the spec file.
+
+        resources = {}
+
+        self._logger.event(
+            "Do you want to apply this changes?", bold=True, previous_line=True
+        )
+        if not confirm("Insert choice: "):
+            self._logger.event("No actions performed")
+            return
+
+        # FIXME: dependency_graph order in delete
+        # First remove the state items not present in the spec file
+        for key in self._state.all():
+            if key not in self._specs:
+                data = self._state.get(key)
+
+                resource = self._create_resource(data)
+
+                resource.delete()
+                self._logger.resource("Resource deleted", resource)
+
+                self._state.delete(key)
+                self._state.save()
+
+        for key in self._specs_order:
+            # If this resource does not exist yet
+            if not self._state.contains(key):
+                data = self._specs[key]
+
+                resource = self._create_resource(data)
+                resources[key] = resource
+
+                # Replace each reference of this resource
+                self._update_references(resource, resources)
+
+                resource.create()
+                self._logger.resource("Resource created", resource)
+
+                self._state.add(resource.key, resource.dump())
+                self._state.save()
+
+            # Already exists in the engine
+            else:
+                state_data = self._state.get(key)
+                state_resource = self._create_resource(state_data)
+
+                spec_data = self._specs[key]
+                spec_resource = self._create_resource(spec_data)
+
+                # Replace each reference of this resource
+                self._update_references(spec_resource, resources)
+
+                diff = Diff(
+                    new_arguments=spec_resource.arguments,
+                    old_arguments=state_resource.arguments,
+                )
+
+                resources[key] = state_resource
+
+                if diff:
+                    state_resource.update_arguments(spec_resource.arguments)
+                    state_resource.update()
+                    self._logger.resource("Resource updated", state_resource)
+
+                    self._state.update(key, state_resource.dump())
+                    self._state.save()

splight_cli-4.10.0/splight_cli/solution/resource/models/__init__.py

@@ -0,0 +1,25 @@
+from .alert import Alert
+from .asset import Asset
+from .attribute import Attribute
+from .base import Resource
+from .component import Component
+from .component_object import ComponentObject
+from .file import File
+from .function import Function
+from .metadata import Metadata
+from .routine import Routine
+from .secret import Secret
+
+__all__ = [
+    "Alert",
+    "Component",
+    "ComponentObject",
+    "Routine",
+    "Resource",
+    "Asset",
+    "Secret",
+    "Attribute",
+    "Metadata",
+    "File",
+    "Function",
+]

splight_cli-4.10.0/splight_cli/solution/resource/models/alert.py

@@ -0,0 +1,14 @@
+from splight_lib.models import Alert as AlertSchema
+from splight_lib.models.base import SplightDatabaseBaseModel
+
+from splight_cli.solution.resource.models.base import Resource
+
+
+# FIXME: Sometimes the API mixes up the order of the alert items after
+# creating the resource, this causes the plan to show a diff.
+# Another apply will fix it, but would be nice if we always sorted new
+# and older arguments by name.
+# Should be done by overriding init and after refresh retrieves the new
+# arguments.
+class Alert(Resource):
+    _schema: SplightDatabaseBaseModel = AlertSchema

splight_cli-4.10.0/splight_cli/solution/resource/models/asset.py

@@ -0,0 +1,8 @@
+from splight_lib.models import Asset as AssetSchema
+from splight_lib.models.base import SplightDatabaseBaseModel
+
+from splight_cli.solution.resource.models.base import Resource
+
+
+class Asset(Resource):
+    _schema: SplightDatabaseBaseModel = AssetSchema

splight_cli-4.10.0/splight_cli/solution/resource/models/attribute.py

@@ -0,0 +1,8 @@
+from splight_lib.models import Attribute as AttributeSchema
+from splight_lib.models.base import SplightDatabaseBaseModel
+
+from splight_cli.solution.resource.models.base import Resource
+
+
+class Attribute(Resource):
+    _schema: SplightDatabaseBaseModel = AttributeSchema