s1-shift-left-cli 0.4.0__py3-none-any.whl

s1_cns_cli/s1graph/terraform/module_loading/loaders/registry_loader.py

@@ -0,0 +1,210 @@
+import os
+import logging
+from http import HTTPStatus
+from typing import List, Dict
+
+import requests
+from requests.exceptions import HTTPError
+from urllib.parse import urlparse
+
+from s1_cns_cli.s1graph.common.models.consts import TFC_HOST_NAME
+from s1_cns_cli.s1graph.common.goget.registry.get_registry import RegistryGetter
+from s1_cns_cli.s1graph.common.util.http_utils import DEFAULT_TIMEOUT
+from s1_cns_cli.s1graph.terraform.module_loading.content import ModuleContent
+from s1_cns_cli.s1graph.terraform.module_loading.loader import ModuleLoader
+from s1_cns_cli.s1graph.terraform.module_loading.loaders.versions_parser import (
+    order_versions_in_descending_order,
+    get_version_constraints
+)
+from s1_cns_cli.s1graph.terraform.module_loading.module_params import ModuleParams
+
+# https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http
+MODULE_ARCHIVE_EXTENSIONS = ["zip", "tar.bz2", "tar.gz", "tgz", "tar.xz", "txz"]
+
+
+class RegistryLoader(ModuleLoader):
+    modules_versions_cache: Dict[str, List[str]] = {}  # noqa: CCE003  # public data
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def discover(self, module_params):
+        module_params.tf_host_name = os.getenv("TF_HOST_NAME", TFC_HOST_NAME)
+        module_params.token = os.getenv("TF_REGISTRY_TOKEN", "")
+        tfc_token = os.getenv("TFC_TOKEN")
+        if tfc_token:
+            self.logger.warn("Environment variable TFC_TOKEN will be deprecated in the future. Please use TF_REGISTRY_TOKEN instead.")
+            module_params.token = tfc_token
+
+    def _is_matching_loader(self, module_params: ModuleParams) -> bool:
+        # https://developer.hashicorp.com/terraform/language/modules/sources#github
+        if module_params.module_source.startswith(("github.com", "bitbucket.org", "git::", "git@github.com")):
+            return False
+        self._process_inner_registry_module(module_params)
+        # determine tf api endpoints
+        self._determine_tf_api_endpoints(module_params)
+        # If versions for a module are cached, determine the best version and return True.
+        # If versions are not cached, get versions, then determine the best version and return True.
+        # Best version needs to be determined here for setting most accurate dest_dir.
+        if module_params.tf_modules_versions_endpoint in RegistryLoader.modules_versions_cache.keys():
+            module_params.best_version = self._find_best_version(module_params)
+            return True
+        if not self._cache_available_versions(module_params):
+            return False
+        module_params.best_version = self._find_best_version(module_params)
+        if not module_params.inner_module:
+            module_params.dest_dir = os.path.join(module_params.root_dir, module_params.external_modules_folder_name,
+                                                  module_params.tf_host_name, *module_params.module_source.split("/"),
+                                                  module_params.best_version)
+        if os.path.exists(module_params.dest_dir):
+            return True
+        # verify cache again after refresh
+        if module_params.tf_modules_versions_endpoint in RegistryLoader.modules_versions_cache.keys():
+            return True
+        return False
+
+    def _load_module(self, module_params: ModuleParams) -> ModuleContent:
+        if module_params.best_version:
+            best_version = module_params.best_version
+        else:
+            if self._cache_available_versions(module_params):
+                module_params.best_version = self._find_best_version(module_params)
+        if os.path.exists(module_params.dest_dir):
+            return ModuleContent(dir=module_params.dest_dir)
+
+        request_download_url = "/".join((module_params.tf_modules_endpoint, module_params.module_source, best_version, "download"))
+        logging.debug(f"Best version for {module_params.module_source} is {best_version} based on the version constraint {module_params.version}.")
+        logging.debug(f"Module download url: {request_download_url}")
+        try:
+            response = requests.get(
+                url=request_download_url,
+                headers={"Authorization": f"Bearer {module_params.token}"} if module_params.token else None,
+                timeout=DEFAULT_TIMEOUT
+            )
+            response.raise_for_status()
+        except HTTPError as e:
+            self.logger.warning(e)
+            if response.status_code != HTTPStatus.OK and response.status_code != HTTPStatus.NO_CONTENT:
+                return ModuleContent(dir=None)
+        # https://www.terraform.io/registry/api-docs#download-source-code-for-a-specific-module-version
+        module_download_url = response.headers.get('X-Terraform-Get', '')
+        self.logger.debug(f"X-Terraform-Get: {module_download_url}")
+        module_download_url = self._normalize_module_download_url(module_params, module_download_url)
+        self.logger.debug(f"Cloning module from normalized url {module_download_url}")
+        if self._is_download_url_archive(module_download_url):
+            try:
+                registry_getter = RegistryGetter(module_download_url)
+                registry_getter.temp_dir = module_params.dest_dir
+                registry_getter.do_get()
+                return_dir = module_params.dest_dir
+            except Exception as e:
+                str_e = str(e)
+                if 'File exists' not in str_e and 'already exists and is not an empty directory' not in str_e:
+                    self.logger.error(f"failed to get {module_params.module_source} because of {e}")
+                    return ModuleContent(dir=None, failed_url=module_params.module_source)
+            if module_params.inner_module:
+                return_dir = os.path.join(module_params.dest_dir, module_params.inner_module)
+            return ModuleContent(dir=return_dir)
+        else:
+            return ModuleContent(dir=None, next_url=response.headers.get("X-Terraform-Get", ""))
+
+    def _find_module_path(self, module_params: ModuleParams) -> str:
+        # to determine the exact path here would be almost a duplicate of the git_loader functionality
+        return ""
+
+    def _find_best_version(self, module_params: ModuleParams) -> str:
+        versions_by_size = RegistryLoader.modules_versions_cache.get(module_params.tf_modules_versions_endpoint, [])
+        if module_params.version == "latest":
+            module_params.version = versions_by_size[0]
+        version_constraints = get_version_constraints(module_params.version)
+        num_of_matches = 0
+        for version in versions_by_size:
+            for version_constraint in version_constraints:
+                if not version_constraint.versions_matching(version):
+                    break
+                else:
+                    num_of_matches += 1
+            if num_of_matches == len(version_constraints):
+                return version
+            else:
+                num_of_matches = 0
+        return "latest"
+
+    def _cache_available_versions(self, module_params: ModuleParams) -> bool:
+        # Get all available versions for a module in the registry and cache them.
+        # Returns False on failure.
+        try:
+            response = requests.get(
+                url=module_params.tf_modules_versions_endpoint,
+                headers={"Authorization": f"Bearer {module_params.token}"} if module_params.token else None,
+                timeout=DEFAULT_TIMEOUT,
+            )
+            response.raise_for_status()
+            available_versions = [
+                v.get("version") for v in response.json().get("modules", [{}])[0].get("versions", {})
+            ]
+            RegistryLoader.modules_versions_cache[module_params.tf_modules_versions_endpoint] = order_versions_in_descending_order(
+                available_versions)
+            return True
+        except HTTPError as e:
+            self.logger.debug(e)
+            return False
+
+    def _process_inner_registry_module(self, module_params: ModuleParams) -> None:
+        # Check if the source has '//' in it. If it does, it indicates a reference for an inner module.
+        # Example: "terraform-aws-modules/security-group/aws//modules/http-80" =>
+        #    module_source = terraform-aws-modules/security-group/aws
+        #    dest_dir = modules/http-80
+        module_source_components = module_params.module_source.split("//")
+        if len(module_source_components) > 1:
+            module_params.module_source = module_source_components[0]
+            module_params.dest_dir = module_params.dest_dir.split("//")[0]
+            module_params.inner_module = module_source_components[1]
+
+    def _determine_tf_api_endpoints(self, module_params: ModuleParams) -> None:
+        """
+        Determines terraform registry endpoints - tf_host_name, tf_modules_endpoint, tf_modules_versions_endpoint
+        """
+        if module_params.module_source.startswith(module_params.tf_host_name):
+            # check if module source supports native Terraform services
+            # https://www.terraform.io/internals/remote-service-discovery#remote-service-discovery
+            module_params.module_source = module_params.module_source.replace(f"{module_params.tf_host_name}/", "")
+            try:
+                response = requests.get(
+                    url=f"https://{module_params.tf_host_name}/.well-known/terraform.json",
+                    timeout=DEFAULT_TIMEOUT,
+                )
+                response.raise_for_status()
+            except HTTPError as e:
+                self.logger.debug(e)
+                if response.status_code != HTTPStatus.OK and response.status_code != HTTPStatus.NO_CONTENT:
+                    return False
+
+            self.logger.debug(f"Service discovery response: {response.json()}")
+            module_params.tf_modules_endpoint = f"https://{module_params.tf_host_name}{response.json().get('modules.v1')}"
+        else:
+            # use terraform cloud host name and url for the public registry
+            module_params.tf_host_name = TFC_HOST_NAME
+            module_params.tf_modules_endpoint = "https://registry.terraform.io/v1/modules"
+        module_params.tf_modules_versions_endpoint = "/".join((module_params.tf_modules_endpoint, module_params.module_source, "versions"))
+
+    def _normalize_module_download_url(self, module_params: ModuleParams, module_download_url) -> str:
+        if not urlparse(module_download_url).netloc:
+            module_download_url = f"https://{module_params.tf_host_name}{module_download_url}"
+        return module_download_url
+
+    @staticmethod
+    def _is_download_url_archive(module_download_url) -> bool:
+        for extension in MODULE_ARCHIVE_EXTENSIONS:
+            if module_download_url.endswith(extension):
+                return True
+        query_params = urlparse(module_download_url).query
+        if query_params:
+            query_params = query_params.split("&")
+            for query_param in query_params:
+                if query_param.startswith("archive="):
+                    return True
+        return False
+
+
+loader = RegistryLoader()

s1_cns_cli/s1graph/terraform/module_loading/loaders/versions_parser.py

@@ -0,0 +1,68 @@
+import re
+from typing import List, Dict, Optional
+
+from s1_cns_cli.s1graph.common.packaging import version
+
+VERSION_REGEX = re.compile(r"^(?P<operator>=|!=|>=|>|<=|<|~>)?\s*(?P<version>[\d.]+-?\w*)$")
+
+
+class VersionConstraint:
+    """
+    A class representing a module version. Enables comparing versions.
+    """
+
+    def __init__(self, constraint_parts: Dict[str, Optional[str]]) -> None:
+        """
+        :param constraint_parts: a dictionary representing a version constraint: {"version": "v1.2.3", "operator": ">="}
+        """
+        self.version = version.parse(constraint_parts.get("version") or "")
+        self.operator = constraint_parts.get("operator") or "="
+
+    def get_max_version_for_most_specific_segment(self) -> version.Version:
+        return version.parse(f"{self.version.major + 1}.0.0")
+
+    def versions_matching(self, other_version_str: str) -> bool:
+        other_version = version.parse(other_version_str)
+        res = {
+            "=": lambda other: other == self.version,
+            "!=": lambda other: other != self.version,
+            ">": lambda other: other > self.version,
+            ">=": lambda other: other >= self.version,
+            "<": lambda other: other < self.version,
+            "<=": lambda other: other <= self.version,
+            "~>": lambda other: self.version <= other < self.get_max_version_for_most_specific_segment(),
+        }[self.operator](other_version)
+        return res
+
+    def __str__(self) -> str:
+        return f"{self.operator}{self.version}"
+
+
+def get_version_constraints(raw_version: str) -> List[VersionConstraint]:
+    """
+    :param raw_version: A string representation of a version, e.g: "~> v1.2.3"
+    :return: VersionConstraint instance
+    """
+    raw_version = raw_version.replace(" ", "")
+    raw_version_constraints = raw_version.split(",")
+    version_constraints = []
+    for constraint in raw_version_constraints:
+        match = re.search(VERSION_REGEX, constraint)
+        if match:
+            constraint_parts = match.groupdict()
+            version_constraints.append(VersionConstraint(constraint_parts))
+    return version_constraints
+
+
+def order_versions_in_descending_order(versions_strings: List[str]) -> List[str]:
+    """
+    :param versions_strings: array of string versions: ["v1.2.3", "v1.2.4"...]
+    :return: A sorted array of versions in descending order
+    """
+    for iter_num in range(len(versions_strings) - 1, 0, -1):
+        for idx in range(iter_num):
+            if version.parse(versions_strings[idx]) < version.parse(versions_strings[idx + 1]):
+                temp = versions_strings[idx]
+                versions_strings[idx] = versions_strings[idx + 1]
+                versions_strings[idx + 1] = temp
+    return versions_strings

s1_cns_cli/s1graph/terraform/module_loading/module_finder.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import logging
+import os
+import re
+from typing import List, Callable
+
+from s1_cns_cli.s1graph.common.parallelizer.parallel_runner import parallel_runner
+from s1_cns_cli.s1graph.terraform.module_loading.registry import module_loader_registry
+
+
+class ModuleDownload:
+    def __init__(self, source_dir: str) -> None:
+        self.source_dir = source_dir
+        self.module_link: str | None = None
+        self.version: str | None = None
+
+    def __str__(self):
+        return f"{self.source_dir} -> {self.module_link} ({self.version})"
+
+    @property
+    def address(self):
+        return f'{self.module_link}:{self.version}'
+
+
+def find_modules(path: str) -> List[ModuleDownload]:
+    modules_found = []
+    for root, _, full_file_names in os.walk(path):
+        for file_name in full_file_names:
+            if not file_name.endswith('.tf'):
+                continue
+            with open(os.path.join(path, root, file_name)) as f:
+                try:
+                    in_module = False
+                    curr_md = None
+                    for line in f:
+                        if line.strip().startswith('#'):
+                            continue
+                        if not in_module:
+                            if line.startswith('module'):
+                                in_module = True
+                                curr_md = ModuleDownload(os.path.dirname(os.path.join(root, file_name)))
+                                continue
+                        if in_module:
+                            if line.startswith('}'):
+                                in_module = False
+                                if curr_md.module_link is None:
+                                    logging.warning(f'A module at {curr_md.source_dir} had no source, skipping')
+                                else:
+                                    modules_found.append(curr_md)
+                                curr_md = None
+                                continue
+
+                            match = re.match(re.compile('.*\\bsource\\s*=\\s*"(?P<LINK>.*)"'), line)
+                            if match:
+                                curr_md.module_link = match.group('LINK')
+                                continue
+
+                            match = re.match(re.compile('.*\\bversion\\s*=\\s*"(?P<operator>=|!=|>=|>|<=|<|~>)?\\s*(?P<version>[\\d.]+-?\\w*)"'), line)
+                            if match:
+                                curr_md.version = f"{match.group('operator')}{match.group('version')}" if match.group('operator') else match.group('version')
+                except (UnicodeDecodeError, FileNotFoundError) as e:
+                    logging.warning(f"Skipping {os.path.join(path, root, file_name)} because of {e}")
+                    continue
+
+    return modules_found
+
+
+def should_download(path: str) -> bool:
+    return not (path.startswith('./') or path.startswith('../') or path.startswith('/'))
+
+
+def load_tf_modules(path: str, should_download_module: Callable[[str], bool] = should_download, run_parallel=False,
+                    modules_to_load: List[ModuleDownload] = None):
+    module_loader_registry.root_dir = path
+    if not modules_to_load:
+        modules_to_load = find_modules(path)
+
+    def _download_module(m):
+        if should_download_module(m.module_link):
+            logging.info(f'Downloading module {m.address}')
+            try:
+                content = module_loader_registry.load(m.source_dir, m.module_link,
+                                                      "latest" if not m.version else m.version)
+                if content is None or not content.loaded():
+                    log_message = f'Failed to download module {m.address}'
+                    if not module_loader_registry.download_external_modules:
+                        log_message += ' (for external modules, the --download-external-modules flag is required)'
+                    logging.warning(log_message)
+            except Exception as e:
+                logging.warning(f"Unable to load module ({m.address}): {e}")
+
+    # To avoid duplicate work, we need to get the distinct module sources
+    distinct_modules = list({m.address: m for m in modules_to_load}.values())
+
+    if run_parallel:
+        list(parallel_runner.run_function(_download_module, distinct_modules))
+    else:
+        for m in distinct_modules:
+            _download_module(m)

s1_cns_cli/s1graph/terraform/module_loading/module_params.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class ModuleParams:
+    def __init__(self, root_dir: str, current_dir: str, source: str, source_version: Optional[str], dest_dir: str,
+                 external_modules_folder_name: str, inner_module: Optional[str] = None):
+        self.root_dir: str = root_dir
+        self.current_dir: str = current_dir
+        self.module_source: str = source
+        self.version: Optional[str] = source_version
+        self.dest_dir: str = dest_dir
+        self.external_modules_folder_name: str = external_modules_folder_name
+        self.inner_module: Optional[str] = inner_module
+
+        self.token: Optional[str] = None
+        self.username: Optional[str] = None
+        self.vcs_base_url: Optional[str] = None
+        self.module_source_prefix: Optional[str] = None
+        self.best_version: Optional[str] = None
+
+        # terraform cloud / enterprise specific params
+        self.tf_host_name: Optional[str] = None
+        self.tf_modules_endpoint: Optional[str] = None
+        self.tf_modules_versions_endpoint: Optional[str] = None

s1_cns_cli/s1graph/terraform/module_loading/registry.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import logging
+import os
+import hashlib
+from typing import Optional, List, TYPE_CHECKING, Set, Dict
+
+from s1_cns_cli.s1graph.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR
+from s1_cns_cli.s1graph.terraform.module_loading.content import ModuleContent
+from s1_cns_cli.s1graph.terraform.module_loading.module_params import ModuleParams
+
+if TYPE_CHECKING:
+    from s1_cns_cli.s1graph.terraform.module_loading.loader import ModuleLoader
+
+
+class ModuleLoaderRegistry:
+    loaders: List["ModuleLoader"] = []  # noqa: CCE003
+    module_content_cache: Dict[str, Optional[ModuleContent]] = {}  # noqa: CCE003
+
+    def __init__(
+        self, download_external_modules: bool = False, external_modules_folder_name: str = DEFAULT_EXTERNAL_MODULES_DIR
+    ) -> None:
+        self.logger = logging.getLogger(__name__)
+        self.download_external_modules = download_external_modules
+        self.external_modules_folder_name = external_modules_folder_name
+        self.failed_urls_cache: Set[str] = set()
+        self.root_dir = ""  # root dir for storing external modules
+
+    def load(self, current_dir: str, source: str, source_version: Optional[str]) -> ModuleContent:
+        """
+Search all registered loaders for the first one which is able to load the module source type. For more
+information, see `loader.ModuleLoader.load`.
+        """
+        module_address = f'{source}:{source_version}'
+        if module_address in self.module_content_cache:
+            logging.debug(f'Used the cache for module {module_address}')
+            return self.module_content_cache[module_address]
+        else:
+            logging.debug(f'Cache miss for {module_address}')
+
+        if os.name == 'nt':
+            # For windows, due to limitations in the allowed characters for path names, the hash of the source is used.
+            # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
+            source_hash = hashlib.md5(source.encode())  # nosec
+            local_dir = os.path.join(self.root_dir, self.external_modules_folder_name, source_hash.hexdigest())
+        else:
+            local_dir = os.path.join(self.root_dir, self.external_modules_folder_name, source)
+        inner_module = ""
+        next_url = source
+        last_exception = None
+        content = ModuleContent(None)
+        while next_url:
+            source = next_url
+            next_url = ""
+            if source in self.failed_urls_cache:
+                break
+            for loader in self.loaders:
+                if not self.download_external_modules and loader.is_external:
+                    continue
+                try:
+                    module_params = ModuleParams(root_dir=self.root_dir,
+                                                 current_dir=current_dir,
+                                                 source=source,
+                                                 source_version=source_version,
+                                                 dest_dir=local_dir,
+                                                 external_modules_folder_name=self.external_modules_folder_name,
+                                                 inner_module=inner_module)
+                    content = loader.load(module_params)
+                except Exception as e:
+                    logging.warning(f'Module {module_address} failed to load via {loader.__class__}')
+                    last_exception = e
+                    continue
+                if content.next_url:
+                    next_url = content.next_url
+                    if module_params.inner_module:
+                        local_dir = module_params.dest_dir
+                        inner_module = module_params.inner_module
+                    break
+                if content is None:
+                    continue
+                elif not content.loaded():
+                    if content.failed_url:
+                        self.failed_urls_cache.add(content.failed_url)
+                    self.module_content_cache[module_address] = ModuleContent(None)
+                    continue
+                else:
+                    self.module_content_cache[module_address] = content
+                    return content
+
+        if last_exception is not None:
+            raise last_exception
+
+        self.module_content_cache[module_address] = content
+        return content
+
+    def register(self, loader: "ModuleLoader") -> None:
+        self.loaders.append(loader)
+
+    def reset_module_content_cache(self) -> None:
+        self.module_content_cache = {}
+
+    def clear_all_loaders(self) -> None:
+        self.loaders.clear()
+
+
+module_loader_registry = ModuleLoaderRegistry()

s1_cns_cli/s1graph/terraform/modules/__init__.py

@@ -0,0 +1,2 @@
+from s1_cns_cli.s1graph.terraform.modules.module_utils import *  # noqa
+from s1_cns_cli.s1graph.terraform.modules.module_objects import *  # noqa

s1_cns_cli/s1graph/terraform/modules/module_objects.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+import json
+from typing import Optional, Any
+
+
+class TFModule:
+    __slots__ = ("path", "name", "foreach_idx", "nested_tf_module")
+
+    def __init__(self, path: str, name: str, nested_tf_module: Optional[TFModule] = None,
+                 foreach_idx: Optional[int | str] = None) -> None:
+        self.path = path
+        self.name = name
+        self.foreach_idx = foreach_idx
+        self.nested_tf_module = nested_tf_module
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, TFModule):
+            return False
+        return self.path == other.path and self.name == other.name and self.nested_tf_module == other.nested_tf_module and self.foreach_idx == other.foreach_idx
+
+    def __lt__(self, other: Any) -> bool:
+        if not isinstance(other, TFModule):
+            return False
+        return (self.path, self.name, self.nested_tf_module, self.foreach_idx) < (
+            other.path, other.name, other.nested_tf_module, other.foreach_idx)
+
+    def __repr__(self) -> str:
+        return f'path:{self.path}, name:{self.name}, nested_tf_module:{self.nested_tf_module}, foreach_idx:{self.foreach_idx}'
+
+    def __hash__(self) -> int:
+        return hash((self.path, self.name, self.nested_tf_module, self.foreach_idx))
+
+    def __iter__(self):
+        yield from {
+            "path": self.path,
+            "name": self.name,
+            "foreach_idx": self.foreach_idx,
+            "nested_tf_module": dict(self.nested_tf_module) if self.nested_tf_module else None
+        }.items()
+
+    def __str__(self):
+        from s1_cns_cli.s1graph.common.util.json_utils import CustomJSONEncoder
+        return json.dumps(dict(self), cls=CustomJSONEncoder)
+
+    @staticmethod
+    def from_json(json_dct):
+        return TFModule(path=json_dct['path'], name=json_dct['name'], foreach_idx=json_dct['foreach_idx'],
+                        nested_tf_module=TFModule.from_json(json_dct['nested_tf_module']) if json_dct.get(
+                            'nested_tf_module') else None)
+
+
+class TFDefinitionKey:
+    __slots__ = ("tf_source_modules", "file_path")
+
+    def __init__(self, file_path: str, tf_source_modules: Optional[TFModule] = None) -> None:
+        self.tf_source_modules = tf_source_modules
+        self.file_path = file_path
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, TFDefinitionKey):
+            return False
+        return self.tf_source_modules == other.tf_source_modules and self.file_path == other.file_path
+
+    def __lt__(self, other: Any) -> bool:
+        if not isinstance(other, TFDefinitionKey):
+            return False
+        return (self.file_path, self.tf_source_modules) < (other.file_path, other.tf_source_modules)
+
+    def __repr__(self) -> str:
+        return f'tf_source_modules:{self.tf_source_modules}, file_path:{self.file_path}'
+
+    def __hash__(self) -> int:
+        return hash((self.file_path, self.tf_source_modules))
+
+    def __iter__(self):
+        yield from {
+            "file_path": self.file_path,
+            "tf_source_modules": dict(self.tf_source_modules) if self.tf_source_modules else None
+        }.items()
+
+    def __str__(self):
+        from s1_cns_cli.s1graph.common.util.json_utils import CustomJSONEncoder
+        return json.dumps(self.to_json(), cls=CustomJSONEncoder)
+
+    def to_json(self):
+        to_return = {"file_path": self.file_path, "tf_source_modules": None}
+        if self.tf_source_modules:
+            to_return["tf_source_modules"] = dict(self.tf_source_modules)
+        return to_return
+
+    @staticmethod
+    def from_json(json_dct):
+        return TFDefinitionKey(file_path=json_dct['file_path'],
+                               tf_source_modules=TFModule.from_json(json_dct['tf_source_modules']))