prepare-assignment 0.4.2__tar.gz → 0.4.4__tar.gz

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: prepare-assignment
-Version: 0.4.2
+Version: 0.4.4
 Summary: Prepare assignment
 Author-email: Bonajo <m.bonajo@fontys.nl>
 License: MIT

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/prepare_assignment/core/preparer.py

@@ -12,7 +12,7 @@ from virtualenv import cli_run
 
 from prepare_assignment.core.versions import COMMIT_HASH_RE, get_git_url, resolve_version
 from prepare_assignment.core.validator import validate_task_definition, validate_tasks, load_yaml, \
-    validate_default_values
+    validate_default_values, validate_unique_ids
 from prepare_assignment.data.errors import DependencyError, ValidationError, PrepareTaskError
 from prepare_assignment.data.task_definition import TaskDefinition, CompositeTaskDefinition, \
     PythonTaskDefinition, ValidableTask
@@ -154,10 +154,19 @@ def __sub_tasks(task: TaskDefinition) -> List[Any]:
     return [step for step in task.tasks if step.get("uses", None) is not None]
 
 
+def __validate_sub_task_ids(props: TaskProperties, task: TaskDefinition) -> None:
+    if isinstance(task, CompositeTaskDefinition):
+        validate_unique_ids(str(props.definition_path), f"task '{task.name}'", task.tasks)
+
+
 def __load_task_from_disk(props: TaskProperties, parsed: Dict[str, ValidableTask]) -> None:
     logger.debug(f"Task '{props}' is already available, loading from disk")
     task_yaml = load_yaml(props.definition_path)
     task = TaskDefinition.of(task_yaml, props.task_path)
+    try:
+        __validate_sub_task_ids(props, task)
+    except ValidationError as e:
+        raise PrepareTaskError(f"Unable to prepare task '{props}'", e) from e
     # Always rebuild the schema, so schemas generated by older versions of prepare are updated
     json_schema = __build_json_schema(props, task)
     with open(__schema_path(props), "r", encoding="utf-8") as handle:
@@ -181,6 +190,7 @@ def __prepare_task(props: TaskProperties) -> ValidableTask:
         task_yaml = validate_task_definition(props.definition_path)
         task: TaskDefinition = TaskDefinition.of(task_yaml, props.task_path)
         validate_default_values(task)
+        __validate_sub_task_ids(props, task)
         if isinstance(task, PythonTaskDefinition):
             main_path = os.path.join(props.repo_path, Path(task.main))  # type: ignore
             if not Path(main_path).resolve().is_relative_to(props.repo_path.resolve()):

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/prepare_assignment/core/task_handler.py

@@ -9,7 +9,7 @@ from treelib import Tree
 from prepare_assignment.core.preparer import __prepare_tasks
 from prepare_assignment.core.versions import get_git_url, is_fixed_version
 from prepare_assignment.data.task_definition import TaskDefinition
-from prepare_assignment.data.task_properties import TaskProperties
+from prepare_assignment.data.task_properties import TaskProperties, directory_to_version
 from prepare_assignment.utils.files import remove_tree
 from prepare_assignment.utils.dependency import get_dependencies
 from prepare_assignment.utils.paths import get_tasks_path
@@ -88,7 +88,7 @@ def ls() -> None:
             version_path = os.path.join(org_path, task)
             versions = sorted(os.listdir(version_path))
             for version in versions:
-                tree.create_node(version, parent=task)
+                tree.create_node(directory_to_version(version), parent=task)
     t = tree.show(stdout=False)
     print(t)
 

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/prepare_assignment/core/validator.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 import os
 from pathlib import Path
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
@@ -37,9 +37,32 @@ def validate_prepare(prepare_file: str, prepare: Dict[str, Any]) -> None:
     except ValidationError as ve:
         message = f"Error in: {prepare_file}, unable to verify '{ve.json_path}'\n\t -> {ve.message}"
         raise VE(message)
+    for job, steps in prepare["jobs"].items():
+        validate_unique_ids(prepare_file, f"job '{job}'", steps)
     logger.debug("✓ Prepare file is valid")
 
 
+def validate_unique_ids(file: str, where: str, steps: List[Dict[str, Any]]) -> None:
+    """
+    Validate that the ids of the steps are unique, the outputs of a step are stored under its id
+    NOTE: steps without an id are not checked
+    :param file: path/name of the file that contains the steps
+    :param where: the job or composite task that contains the steps, used in the error message
+    :param steps: the steps of the job or composite task
+    :return: None
+    :raises: ValidationError: if multiple steps have the same id
+    """
+    names: Dict[str, List[str]] = {}
+    for step in steps:
+        step_id = step.get("id", None)
+        if step_id is not None:
+            names.setdefault(step_id, []).append(step.get("name", ""))
+    for step_id, step_names in names.items():
+        if len(step_names) > 1:
+            quoted = ", ".join(f"'{name}'" for name in step_names)
+            raise VE(f"Error in: {file}, {where} has multiple steps with id '{step_id}': {quoted}")
+
+
 def validate_tasks(file: str, task: Dict[str, Any], json_schema: Any) -> None:
     """
     Validate all tasks based on their respective json schemas

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/prepare_assignment/data/task_properties.py

@@ -18,7 +18,14 @@ class TaskProperties:
 
     @cached_property
     def task_path(self) -> Path:
-        return Path(os.path.join(tasks_path, self.organization, self.name, self.version))
+        return Path(os.path.join(tasks_path, self.organization, self.name, self.version_directory))
+
+    @property
+    def version_directory(self) -> str:
+        """
+        The version as a single directory name, a branch can contain a slash (e.g. fix/something)
+        """
+        return version_to_directory(self.version)
 
     @cached_property
     def repo_path(self) -> Path:
@@ -38,19 +45,21 @@ class TaskProperties:
 
     @classmethod
     def of(cls, task: str) -> TaskProperties:
-        parts = task.split("/")
+        # Everything after the first '@' is the version, a branch can contain a slash
+        name, separator, version = task.partition("@")
+        if not separator:
+            version = "latest"
+        parts = name.split("/")
         if len(parts) > 2:
             raise ValueError("Tasks cannot have more than one slash")
         elif len(parts) == 1:
             parts.insert(0, "prepare-assignment")
-        organization: str = parts[0]
-        name = parts[1]
-        split = name.split("@")
-        version: str = "latest"
-        task_name: str = name
-        if len(split) > 2:
-            raise ValueError("Cannot have multiple '@' symbols in the name")
-        elif len(split) == 2:
-            task_name = split[0]
-            version = split[1]
-        return cls(organization, task_name, version)
+        return cls(parts[0], parts[1], version)
+
+
+def version_to_directory(version: str) -> str:
+    return version.replace("%", "%25").replace("/", "%2F")
+
+
+def directory_to_version(directory: str) -> str:
+    return directory.replace("%2F", "/").replace("%25", "%")

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/prepare_assignment/utils/tasks.py

@@ -2,7 +2,7 @@ import os
 from typing import List
 
 from prepare_assignment.data.task_definition import TaskDefinition
-from prepare_assignment.data.task_properties import TaskProperties
+from prepare_assignment.data.task_properties import TaskProperties, directory_to_version
 from prepare_assignment.utils.paths import get_tasks_path
 from prepare_assignment.utils.yml_loader import YAML_LOADER
 
@@ -20,7 +20,7 @@ def get_all_tasks() -> List[TaskProperties]:
             version_path = os.path.join(org_path, task)
             versions = os.listdir(version_path)
             for version in versions:
-                tasks.append(TaskProperties.of(f"{org}/{task}@{version}"))
+                tasks.append(TaskProperties(org, task, directory_to_version(version)))
     return tasks
 
 

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "prepare-assignment"
-version = "0.4.2"
+version = "0.4.4"
 description = "Prepare assignment"
 authors = [{name = "Bonajo", email = "m.bonajo@fontys.nl"}]
 license = {text = "MIT"}

prepare_assignment-0.4.4/tests/conftest.py

@@ -0,0 +1,25 @@
+import logging
+from typing import Iterator
+
+import pytest
+
+# Loggers that prepare() configures (level, handlers), see prepare_assignment/core/main.py
+_LOGGERS = ("prepare_assignment", "tasks")
+
+
+@pytest.fixture(autouse=True)
+def reset_loggers() -> Iterator[None]:
+    """
+    Restore the prepare loggers after every test. prepare() sets their level (e.g. ERROR for verbosity 0), which made
+    tests that check log output with caplog fail when they ran after a test that calls prepare().
+    """
+    saved = {}
+    for name in _LOGGERS:
+        logger = logging.getLogger(name)
+        saved[name] = (logger.level, list(logger.handlers), logger.propagate)
+    yield
+    for name, (level, handlers, propagate) in saved.items():
+        logger = logging.getLogger(name)
+        logger.setLevel(level)
+        logger.handlers[:] = handlers
+        logger.propagate = propagate

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/tests/core/test_preparer.py

@@ -283,3 +283,37 @@ def test_outdated_schema_is_updated_on_load(mocker: MockerFixture) -> None:
     spy.assert_not_called()
     with open(schema_file) as handle:
         assert "if" in json.load(handle)["properties"]
+
+
+DUPLICATE_ID_COMPOSITE: Final[Dict[str, Any]] = {
+    'id': 'duplicate',
+    'name': 'duplicate',
+    'description': 'Composite task with a duplicate step id',
+    'runs': {
+        'using': 'composite',
+        'tasks': [{'name': 'first', 'id': 'same', 'run': 'echo 1'},
+                  {'name': 'second', 'id': 'same', 'run': 'echo 2'}]
+    }
+}
+
+
+def test_composite_duplicate_id_is_rejected(mocker: MockerFixture) -> None:
+    __clean_cache()
+    mocker.patch("prepare_assignment.core.preparer.__download_task")
+    mocker.patch("prepare_assignment.core.preparer.validate_task_definition", return_value=DUPLICATE_ID_COMPOSITE)
+    remove = mocker.patch("prepare_assignment.core.preparer.remove_tree")
+    prepare = {'prepare': [{'name': 'duplicate', 'uses': 'duplicate', 'with': {}}]}
+    with pytest.raises(PrepareTaskError) as exc:
+        prepare_tasks("prepare.yml", prepare)
+    assert "multiple steps with id 'same'" in str(exc.value.cause)
+    remove.assert_called()
+
+
+def test_cached_composite_duplicate_id_is_rejected(mocker: MockerFixture) -> None:
+    __clean_cache()
+    mocker.patch("prepare_assignment.core.preparer.__is_installed", return_value=True)
+    mocker.patch("prepare_assignment.core.preparer.load_yaml", return_value=DUPLICATE_ID_COMPOSITE)
+    prepare = {'prepare': [{'name': 'duplicate', 'uses': 'duplicate', 'with': {}}]}
+    with pytest.raises(PrepareTaskError) as exc:
+        prepare_tasks("prepare.yml", prepare)
+    assert "multiple steps with id 'same'" in str(exc.value.cause)

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/tests/core/test_task_handler.py

@@ -93,3 +93,14 @@ def test_load_task_uses_task_path(tasks_dir: Path) -> None:
     props = TaskProperties.of("remove")
     _install_fake(props, "x")
     assert load_task(props).path == props.task_path
+
+
+def test_version_with_slash(tasks_dir: Path, mocker: MockerFixture, capsys: pytest.CaptureFixture) -> None:
+    from prepare_assignment.utils.tasks import get_all_tasks
+    mocker.patch("prepare_assignment.utils.tasks.get_tasks_path", return_value=tasks_dir)
+    mocker.patch("prepare_assignment.core.task_handler.tasks_path", tasks_dir)
+    props = TaskProperties.of("remove@fix/something")
+    _install_fake(props, "branch")
+    assert get_all_tasks() == [props]
+    task_handler.ls()
+    assert "fix/something" in capsys.readouterr().out

{prepare_assignment-0.4.2 → prepare_assignment-0.4.4}/tests/core/test_validator.py

@@ -5,7 +5,8 @@ import pytest
 from pytest_mock import MockerFixture
 
 from prepare_assignment.core.validator import (validate_prepare, validate_tasks, load_yaml,
-                                               validate_task_definition, validate_default_values)
+                                               validate_task_definition, validate_default_values,
+                                               validate_unique_ids)
 from prepare_assignment.data.task_definition import PythonTaskDefinition
 from prepare_assignment.data.errors import ValidationError
 
@@ -179,3 +180,33 @@ def test_validate_default_values_bool_for_integer_invalid() -> None:
     task_copy["inputs"]["num"] = {'description': 'n', 'type': 'integer', 'default': True, 'required': False}
     with pytest.raises(ValidationError):
         validate_default_values(PythonTaskDefinition.of(task_copy, "test.yml"))
+
+
+def test_validate_unique_ids_valid() -> None:
+    steps = [{'name': 'a', 'id': 'a', 'run': 'echo'}, {'name': 'b', 'id': 'b', 'run': 'echo'},
+             {'name': 'no id', 'run': 'echo'}, {'name': 'no id', 'run': 'echo'}]
+    validate_unique_ids("prepare.yml", "job 'build'", steps)
+
+
+def test_validate_unique_ids_duplicate() -> None:
+    steps = [{'name': 'Remove old artefacts', 'id': 'remove', 'uses': 'remove'},
+             {'name': 'other', 'id': 'other', 'uses': 'remove'},
+             {'name': 'Remove old temp directory', 'id': 'remove', 'uses': 'remove'}]
+    with pytest.raises(ValidationError) as exc:
+        validate_unique_ids("prepare.yml", "job 'build'", steps)
+    assert exc.value.message == ("Error in: prepare.yml, job 'build' has multiple steps with id 'remove': "
+                                 "'Remove old artefacts', 'Remove old temp directory'")
+
+
+def test_validate_prepare_duplicate_id() -> None:
+    prepare = {'name': 'Duplicate', 'jobs': {'build': [{'name': 'a', 'id': 'same', 'run': 'echo'},
+                                                       {'name': 'b', 'id': 'same', 'run': 'echo'}]}}
+    with pytest.raises(ValidationError) as exc:
+        validate_prepare("prepare.yml", prepare)
+    assert "job 'build'" in exc.value.message
+
+
+def test_validate_prepare_same_id_in_different_jobs() -> None:
+    prepare = {'name': 'Different jobs', 'jobs': {'build': [{'name': 'a', 'id': 'same', 'run': 'echo'}],
+                                                  'test': [{'name': 'b', 'id': 'same', 'run': 'echo'}]}}
+    validate_prepare("prepare.yml", prepare)

prepare_assignment-0.4.4/tests/data/test_task_properties.py

@@ -0,0 +1,45 @@
+import os
+
+import pytest
+
+from prepare_assignment.data.task_properties import TaskProperties, directory_to_version, version_to_directory
+
+
+@pytest.mark.parametrize("task, expected", [
+    ("remove", TaskProperties("prepare-assignment", "remove", "latest")),
+    ("remove@v1", TaskProperties("prepare-assignment", "remove", "v1")),
+    ("prepare-assignment/remove", TaskProperties("prepare-assignment", "remove", "latest")),
+    ("org/remove@v1.0.0", TaskProperties("org", "remove", "v1.0.0")),
+    ("prc2-assignment@fix/tasks-and-shell", TaskProperties("prepare-assignment", "prc2-assignment",
+                                                           "fix/tasks-and-shell")),
+    ("prepare-assignment/remove@fix/something", TaskProperties("prepare-assignment", "remove", "fix/something")),
+    ("remove@feat/a@b", TaskProperties("prepare-assignment", "remove", "feat/a@b")),
+])
+def test_of(task: str, expected: TaskProperties) -> None:
+    assert TaskProperties.of(task) == expected
+
+
+def test_of_more_than_one_slash_in_name() -> None:
+    with pytest.raises(ValueError):
+        TaskProperties.of("a/b/c@v1")
+
+
+def test_str_keeps_slash_in_version() -> None:
+    assert str(TaskProperties.of("remove@fix/something")) == "prepare-assignment/remove@fix/something"
+
+
+def test_task_path_is_single_directory_for_version() -> None:
+    props = TaskProperties.of("remove@fix/something")
+    assert props.task_path.name == "fix%2Fsomething"
+    assert props.task_path.parent.name == "remove"
+
+
+def test_task_path_unchanged_for_plain_version() -> None:
+    assert TaskProperties.of("remove@v1.0.0").task_path.name == "v1.0.0"
+
+
+@pytest.mark.parametrize("version", ["v1", "fix/something", "a/b/c", "100%", "%2F", "a%2F/b%25"])
+def test_version_directory_round_trip(version: str) -> None:
+    directory = version_to_directory(version)
+    assert os.sep not in directory
+    assert directory_to_version(directory) == version