truss 0.10.0rc1__py3-none-any.whl → 0.60.0__py3-none-any.whl

truss/{patch → truss_handle/patch}/calc_patch.py

@@ -3,12 +3,11 @@ from pathlib import Path
 from typing import Dict, List, Optional, Set
 
 import yaml
-from truss.constants import CONFIG_FILE
-from truss.patch.hash import file_content_hash_str
-from truss.patch.types import TrussSignature
-from truss.server.control.patch.requirement_name_identifier import reqs_by_name
-from truss.server.control.patch.system_packages import system_packages_set
-from truss.server.control.patch.types import (
+
+from truss.base.constants import CONFIG_FILE
+from truss.base.truss_config import ExternalData, TrussConfig
+from truss.base.truss_spec import TrussSpec
+from truss.templates.control.control.helpers.custom_types import (
     Action,
     ConfigPatch,
     DataPatch,
@@ -21,20 +20,19 @@ from truss.server.control.patch.types import (
     PythonRequirementPatch,
     SystemPackagePatch,
 )
-from truss.truss_config import ExternalData, TrussConfig
-from truss.truss_spec import TrussSpec
+from truss.templates.control.control.helpers.truss_patch.requirement_name_identifier import (
+    RequirementMeta,
+)
+from truss.templates.control.control.helpers.truss_patch.system_packages import (
+    system_packages_set,
+)
+from truss.truss_handle.patch.custom_types import ChangedPaths, TrussSignature
+from truss.truss_handle.patch.hash import file_content_hash_str
 from truss.util.path import get_ignored_relative_paths
 
 logger: logging.Logger = logging.getLogger(__name__)
-PYCACHE_IGNORE_PATTERNS = [
-    "**/__pycache__/**/*",
-    "**/__pycache__/**",
-]
-UNPATCHABLE_CONFIG_KEYS = [
-    "live_reload",
-    "python_version",
-    "resources",
-]
+PYCACHE_IGNORE_PATTERNS = ["**/__pycache__/**/*", "**/__pycache__/**"]
+UNPATCHABLE_CONFIG_KEYS = ["live_reload", "python_version", "resources"]
 
 
 def calc_truss_patch(
@@ -62,11 +60,12 @@ def calc_truss_patch(
         ignore_patterns = PYCACHE_IGNORE_PATTERNS
 
     changed_paths = _calc_changed_paths(
-        truss_dir,
-        previous_truss_signature.content_hashes_by_path,
-        ignore_patterns,
+        truss_dir, previous_truss_signature.content_hashes_by_path, ignore_patterns
     )
 
+    new_config = TrussConfig.from_yaml(truss_dir / CONFIG_FILE)
+    prev_config = TrussConfig.from_dict(yaml.safe_load(previous_truss_signature.config))
+
     truss_spec = TrussSpec(truss_dir)
     model_module_path = _relative_to_root(truss_spec.model_module_dir)
     data_dir_path = _relative_to_root(truss_spec.data_dir)
@@ -82,6 +81,7 @@ def calc_truss_patch(
         return _strictly_under(path, [data_dir_path])
 
     patches = []
+
     for path in changed_paths["removed"]:
         if _strictly_under(path, [model_module_path]):
             logger.info(f"Created patch to remove model code file: {path}")
@@ -89,8 +89,7 @@ def calc_truss_patch(
                 Patch(
                     type=PatchType.MODEL_CODE,
                     body=ModelCodePatch(
-                        action=Action.REMOVE,
-                        path=_relative_to(path, model_module_path),
+                        action=Action.REMOVE, path=_relative_to(path, model_module_path)
                     ),
                 )
             )
@@ -110,9 +109,10 @@ def calc_truss_patch(
                 )
             )
         elif _under_unsupported_patch_dir(path):
-            logger.warn(f"Patching not supported for removing {path}")
+            logger.warning(f"Patching not supported for removing {path}")
             return None
 
+    has_calculated_config = False
     for path in changed_paths["added"] + changed_paths["updated"]:
         action = Action.ADD if path in changed_paths["added"] else Action.UPDATE
         if _strictly_under(path, [model_module_path]):
@@ -134,12 +134,16 @@ def calc_truss_patch(
                     ),
                 )
             )
-        elif path == CONFIG_FILE:
-            new_config = TrussConfig.from_yaml(truss_dir / CONFIG_FILE)
-            prev_config = TrussConfig.from_dict(
-                yaml.safe_load(previous_truss_signature.config)
+        elif (
+            path == CONFIG_FILE or _changed_path_is_requirements_file(path, new_config)
+        ) and not has_calculated_config:
+            # we could enter this code block from the requirements file's path
+            # or from the config file's path. In any case, we only want to calculate these
+            # patches once.
+            has_calculated_config = True
+            config_patches = _calc_config_patches(
+                truss_dir, previous_truss_signature, prev_config, new_config
             )
-            config_patches = calc_config_patches(prev_config, new_config)
             if config_patches:
                 logger.info(f"Created patch to {action.value.lower()} config")
             patches.extend(config_patches)
@@ -164,20 +168,32 @@ def calc_truss_patch(
     return patches
 
 
+def _changed_path_is_requirements_file(changed_path: str, new_config: TrussConfig):
+    """
+    _changed_path_is_requirements_file determines if `changed_path` is the same path
+    as the requirements file on a new configuration.
+    """
+    return new_config.requirements_file and Path(changed_path) == Path(
+        new_config.requirements_file
+    )
+
+
 def _calc_changed_paths(
     root: Path,
     previous_root_path_content_hashes: Dict[str, str],
     ignore_patterns: Optional[List[str]],
-) -> Dict[str, List[str]]:
+) -> ChangedPaths:
     """
     TODO(pankaj) add support for directory creation in patch
     """
     root_relative_new_paths = set(
         (str(path.relative_to(root)) for path in root.glob("**/*"))
     )
-    unignored_new_paths = calc_unignored_paths(root_relative_new_paths, ignore_patterns)
+    unignored_new_paths = _calc_unignored_paths(
+        root_relative_new_paths, ignore_patterns
+    )
     previous_root_relative_paths = set(previous_root_path_content_hashes.keys())
-    unignored_prev_paths = calc_unignored_paths(
+    unignored_prev_paths = _calc_unignored_paths(
         previous_root_relative_paths, ignore_patterns
     )
 
@@ -201,9 +217,8 @@ def _calc_changed_paths(
     }
 
 
-def calc_unignored_paths(
-    root_relative_paths: Set[str],
-    ignore_patterns: Optional[List[str]] = None,
+def _calc_unignored_paths(
+    root_relative_paths: Set[str], ignore_patterns: Optional[List[str]] = None
 ) -> Set[str]:
     ignored_paths = set(
         get_ignored_relative_paths(root_relative_paths, ignore_patterns)
@@ -211,8 +226,11 @@ def calc_unignored_paths(
     return root_relative_paths - ignored_paths  # type: ignore
 
 
-def calc_config_patches(
-    prev_config: TrussConfig, new_config: TrussConfig
+def _calc_config_patches(
+    truss_dir: Path,
+    prev_signature: TrussSignature,
+    prev_config: TrussConfig,
+    new_config: TrussConfig,
 ) -> List[Patch]:
     """Calculate patch based on changes to config.
 
@@ -221,11 +239,11 @@ def calc_config_patches(
     """
     try:
         config_patches = _calc_general_config_patches(prev_config, new_config)
-        python_requirement_patches = _calc_python_requirements_patches(
-            prev_config, new_config
+        python_requirements_patches = _calc_requirements_patches(
+            truss_dir, prev_signature, prev_config, new_config
         )
         system_package_patches = _calc_system_packages_patches(prev_config, new_config)
-        return [*config_patches, *python_requirement_patches, *system_package_patches]
+        return [*config_patches, *python_requirements_patches, *system_package_patches]
     except Exception as e:
         logger.error(f"Failed to calculate config patch with exception: {e}")
         raise
@@ -273,10 +291,7 @@ def _calc_env_var_patches(
     removed_items = prev_item_names.difference(new_item_names)
     for removed_item in removed_items:
         patches.append(
-            _mk_env_var_patch(
-                Action.REMOVE,
-                {removed_item: prev_items[removed_item]},
-            )
+            _mk_env_var_patch(Action.REMOVE, {removed_item: prev_items[removed_item]})
         )
     added_items = new_item_names.difference(prev_item_names)
     for added_item in added_items:
@@ -302,41 +317,101 @@ def _calc_external_data_patches(
 
     removed_items = [x for x in prev_items if x not in new_items]
     for removed_item in removed_items:
-        patches.append(
-            _mk_external_data_patch(
-                Action.REMOVE,
-                removed_item,
-            )
-        )
+        patches.append(_mk_external_data_patch(Action.REMOVE, removed_item))
     added_items = [x for x in new_items if x not in prev_items]
     for added_item in added_items:
         patches.append(_mk_external_data_patch(Action.ADD, added_item))
     return patches
 
 
+def _calc_requirements_patches(
+    truss_dir: Path,
+    prev_signature: TrussSignature,
+    prev_config: TrussConfig,
+    new_config: TrussConfig,
+) -> List[Patch]:
+    """
+    requirements patches that are accounted for
+    - contents in the requirements file change
+    - requirements move from config file to requirements file
+    - requirements move from requirements file to config file
+    - requirement file changes (i.e. requirement.txt --> requirements.txt)
+    - requirements in the config file change
+
+    assumes that only one of requirements or requirements_file is present for a given config
+    """
+    prev_requirements = (
+        prev_config.requirements
+        if not prev_config.requirements_file
+        else prev_signature.requirements_file_requirements
+    )
+    new_requirements = (
+        new_config.requirements
+        if not new_config.requirements_file
+        else new_config.load_requirements_from_file(truss_dir)
+    )
+    return _calc_python_requirements_patches(prev_requirements, new_requirements)
+
+
 def _calc_python_requirements_patches(
-    prev_config: TrussConfig, new_config: TrussConfig
+    prev_raw_reqs: List[str], new_raw_reqs: List[str]
 ) -> List[Patch]:
     """Calculate patch based on changes to python requirements.
 
     Empty list means no relevant differences found.
     """
     patches = []
-    prev_reqs = reqs_by_name(prev_config.requirements)
-    prev_req_names = set(prev_reqs.keys())
-    new_reqs = reqs_by_name(new_config.requirements)
-    new_req_names = set(new_reqs.keys())
-    removed_reqs = prev_req_names.difference(new_req_names)
-    for removed_req in removed_reqs:
-        patches.append(_mk_python_requirement_patch(Action.REMOVE, removed_req))
-
-    added_reqs = new_req_names.difference(prev_req_names)
-    for added_req in added_reqs:
-        patches.append(_mk_python_requirement_patch(Action.ADD, new_reqs[added_req]))
-
-    for req in new_req_names.intersection(prev_req_names):
-        if new_reqs[req] != prev_reqs[req]:
-            patches.append(_mk_python_requirement_patch(Action.UPDATE, new_reqs[req]))
+
+    def create_requirement_map(raw_reqs: List[str]) -> Dict[str, RequirementMeta]:
+        req_map = {}
+        for raw_req in raw_reqs:
+            meta = RequirementMeta.from_req(raw_req)
+            req_map[meta.name] = meta
+        return req_map
+
+    prev_reqs_map = create_requirement_map(prev_raw_reqs)
+    new_reqs_map = create_requirement_map(new_raw_reqs)
+    prev_req_names = set(prev_reqs_map.keys())
+    new_req_names = set(new_reqs_map.keys())
+
+    removed_req_names = prev_req_names.difference(new_req_names)
+    for removed_req_name in removed_req_names:
+        removed_req_meta = prev_reqs_map[removed_req_name]
+        if removed_req_meta.is_url_based_requirement:
+            if not removed_req_meta.egg_tag:
+                logger.warning(
+                    f"Url-based requirement `{removed_req_meta.requirement}` is missing egg tag. Ignoring removal. Use `truss push` if you want to remove this requirement."
+                )
+                continue
+            # make sure that the egg tag is included when removing the requirement
+            patches.append(
+                _mk_python_requirement_patch(
+                    Action.REMOVE, removed_req_meta.requirement
+                )
+            )
+        else:
+            patches.append(
+                _mk_python_requirement_patch(Action.REMOVE, removed_req_meta.name)
+            )
+
+    # warn for new reqs
+    added_req_names = new_req_names.difference(prev_req_names)
+    for added_req_name in added_req_names:
+        added_req_meta = new_reqs_map[added_req_name]
+        patches.append(
+            _mk_python_requirement_patch(Action.ADD, added_req_meta.requirement)
+        )
+        if added_req_meta.is_url_based_requirement and not added_req_meta.egg_tag:
+            logger.warning(
+                f"Url-based requirement `{added_req_meta.requirement}` is missing egg tag. Removal will be ignored by `truss watch`"
+            )
+    for req_name in new_req_names.intersection(prev_req_names):
+        if prev_reqs_map[req_name].requirement != new_reqs_map[req_name].requirement:
+            patches.append(
+                _mk_python_requirement_patch(
+                    Action.UPDATE, new_reqs_map[req_name].requirement
+                )
+            )
 
     return patches
 
@@ -363,60 +438,39 @@ def _calc_system_packages_patches(
 
 
 def _mk_config_patch(action: Action, config: dict) -> Patch:
-    return Patch(
-        type=PatchType.CONFIG,
-        body=ConfigPatch(
-            action=action,
-            config=config,
-        ),
-    )
+    return Patch(type=PatchType.CONFIG, body=ConfigPatch(action=action, config=config))
 
 
 # Support for patching data changes yet to be implemented
 def _mk_data_patch(action: Action, item: str, path: str) -> Patch:
     return Patch(
-        type=PatchType.DATA,
-        body=DataPatch(action=action, content=item, path=path),
+        type=PatchType.DATA, body=DataPatch(action=action, content=item, path=path)
     )
 
 
 def _mk_env_var_patch(action: Action, item: dict) -> Patch:
     return Patch(
-        type=PatchType.ENVIRONMENT_VARIABLE,
-        body=EnvVarPatch(
-            action=action,
-            item=item,
-        ),
+        type=PatchType.ENVIRONMENT_VARIABLE, body=EnvVarPatch(action=action, item=item)
     )
 
 
 def _mk_external_data_patch(action: Action, item: Dict[str, str]) -> Patch:
     return Patch(
-        type=PatchType.EXTERNAL_DATA,
-        body=ExternalDataPatch(
-            action=action,
-            item=item,
-        ),
+        type=PatchType.EXTERNAL_DATA, body=ExternalDataPatch(action=action, item=item)
     )
 
 
 def _mk_python_requirement_patch(action: Action, requirement: str) -> Patch:
     return Patch(
         type=PatchType.PYTHON_REQUIREMENT,
-        body=PythonRequirementPatch(
-            action=action,
-            requirement=requirement,
-        ),
+        body=PythonRequirementPatch(action=action, requirement=requirement),
     )
 
 
 def _mk_system_package_patch(action: Action, package: str) -> Patch:
     return Patch(
         type=PatchType.SYSTEM_PACKAGE,
-        body=SystemPackagePatch(
-            action=action,
-            package=package,
-        ),
+        body=SystemPackagePatch(action=action, package=package),
     )
 
 

truss/{types.py → truss_handle/patch/custom_types.py}

@@ -1,42 +1,50 @@
-from dataclasses import dataclass
-from enum import Enum
-from typing import Any, Dict, List
+from dataclasses import dataclass, field
+from typing import Dict, List
 
 from pydantic import BaseModel
-from truss.patch.types import TrussSignature
-from truss.server.control.patch.types import Patch
 
-
-class ModelFrameworkType(Enum):
-    SKLEARN = "sklearn"
-    TENSORFLOW = "tensorflow"
-    KERAS = "keras"
-    PYTORCH = "pytorch"
-    HUGGINGFACE_TRANSFORMER = "huggingface_transformer"
-    XGBOOST = "xgboost"
-    LIGHTGBM = "lightgbm"
-    MLFLOW = "mlflow"
-    CUSTOM = "custom"
+from truss.templates.control.control.helpers.custom_types import Patch
+from truss.util.requirements import parse_requirement_string
 
 
 @dataclass
-class Example:
-    name: str
-    input: Any
+class TrussSignature:
+    """Truss signature stores information for calculating patches for future
+    changes to Truss.
 
-    @staticmethod
-    def from_dict(example_dict):
-        return Example(
-            name=example_dict["name"],
-            input=example_dict["input"],
-        )
+    Currently, it stores hashes of all of the paths in the truss directory excluding the data dir,
+    and the truss config contents. Path hashes allow calculating added/updated/removes
+    paths in future trusses compared to this. Config contents allow calculating
+    config changes, such as add/update/remove of python requirements etc.
+    """
+
+    content_hashes_by_path: Dict[str, str]
+    config: str
+    requirements_file_requirements: List[str] = field(default_factory=list)
 
     def to_dict(self) -> dict:
         return {
-            "name": self.name,
-            "input": self.input,
+            "content_hashes_by_path": self.content_hashes_by_path,
+            "config": self.config,
+            "requirements_file_requirements": self.requirements_file_requirements,
         }
 
+    @staticmethod
+    def from_dict(d) -> "TrussSignature":
+        requirements = []
+        for req in d.get("requirements_file_requirements", []):
+            parsed_req = parse_requirement_string(req)
+            if parsed_req:
+                requirements.append(parsed_req)
+        return TrussSignature(
+            content_hashes_by_path=d["content_hashes_by_path"],
+            config=d["config"],
+            requirements_file_requirements=requirements,
+        )
+
+
+ChangedPaths = Dict[str, List[str]]
+
 
 @dataclass
 class PatchDetails:

truss/{patch → truss_handle/patch}/dir_signature.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Dict, List, Optional
 
-from truss.patch.hash import file_content_hash_str
+from truss.truss_handle.patch.hash import file_content_hash_str
 from truss.util.path import get_unignored_relative_paths_from_root
 
 

truss/truss_handle/patch/hash.py

@@ -0,0 +1,71 @@
+from pathlib import Path
+from typing import Any, List, Optional
+
+from blake3 import blake3
+
+from truss.util.path import get_unignored_relative_paths_from_root
+
+
+def directory_content_hash(
+    root: Path, ignore_patterns: Optional[List[str]] = None
+) -> str:
+    """Calculate content based hash of a filesystem directory.
+
+    Rough algo: Sort all files by path, then take hash of a content stream, where
+    we write path hash to the stream followed by hash of content if path is a file.
+    Note the hash of hash aspect.
+
+    Also, note that name of the root directory is not taken into account, only the contents
+    underneath. The (root) Directory will have the same hash, even if renamed.
+    """
+    hasher = blake3()
+    paths = list(get_unignored_relative_paths_from_root(root, ignore_patterns))
+    paths.sort()
+    for path in paths:
+        hasher.update(str_hash(str(path)))
+        absolute_path = root / path
+        if absolute_path.is_file():
+            hasher.update(file_content_hash(absolute_path))
+    return hasher.hexdigest()
+
+
+def file_content_hash(file: Path) -> bytes:
+    """Calculate blake3 hash of file content.
+    Returns: binary hash of content
+    """
+    return _file_content_hash_loaded_hasher(file).digest()
+
+
+def file_content_hash_str(file: Path) -> str:
+    """Calculate blake3 hash of file content.
+
+    Returns: string hash of content
+    """
+    return _file_content_hash_loaded_hasher(file).hexdigest()
+
+
+def _file_content_hash_loaded_hasher(file: Path) -> Any:
+    hasher = blake3()
+    buffer = bytearray(128 * 1024)
+    mem_view = memoryview(buffer)
+    with file.open("rb") as f:
+        done = False
+        while not done:
+            n = f.readinto(mem_view)
+            if n > 0:
+                hasher.update(mem_view[:n])
+            else:
+                done = True
+    return hasher
+
+
+def str_hash(content: str) -> bytes:
+    hasher = blake3()
+    hasher.update(content.encode("utf-8"))
+    return hasher.digest()
+
+
+def str_hash_str(content: str) -> str:
+    hasher = blake3()
+    hasher.update(content.encode("utf-8"))
+    return hasher.hexdigest()

truss/{patch → truss_handle/patch}/local_truss_patch_applier.py

@@ -3,16 +3,18 @@ import subprocess
 from pathlib import Path
 from typing import List
 
-from truss.server.control.errors import UnsupportedPatch
-from truss.server.control.patch.model_code_patch_applier import apply_code_patch
-from truss.server.control.patch.types import (
+from truss.base.truss_config import TrussConfig
+from truss.templates.control.control.helpers.custom_types import (
     Action,
     ModelCodePatch,
     Patch,
     PythonRequirementPatch,
     SystemPackagePatch,
 )
-from truss.truss_config import TrussConfig
+from truss.templates.control.control.helpers.errors import UnsupportedPatch
+from truss.templates.control.control.helpers.truss_patch.model_code_patch_applier import (
+    apply_code_patch,
+)
 
 
 class LocalTrussPatchApplier:

truss/truss_handle/patch/signature.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+from typing import List, Optional
+
+from truss.base.constants import CONFIG_FILE
+from truss.base.truss_config import TrussConfig
+from truss.truss_handle.patch.custom_types import TrussSignature
+from truss.truss_handle.patch.dir_signature import directory_content_signature
+
+
+def calc_truss_signature(
+    truss_dir: Path, ignore_patterns: Optional[List[str]] = None
+) -> TrussSignature:
+    content_signature = directory_content_signature(truss_dir, ignore_patterns)
+    config_path = truss_dir / CONFIG_FILE
+    with (config_path).open("r") as config_file:
+        config = config_file.read()
+    requirements = TrussConfig.load_requirements_file_from_filepath(config_path)
+    return TrussSignature(
+        content_hashes_by_path=content_signature,
+        config=config,
+        requirements_file_requirements=requirements,
+    )