opensafely 2.1.0__tar.gz → 2.1.2__tar.gz

{opensafely-2.1.0 → opensafely-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensafely
-Version: 2.1.0
+Version: 2.1.2
 Summary: Command line tool for running OpenSAFELY studies locally.
 Author-email: OpenSAFELY <tech@opensafely.org>
 License: GPL-3.0

opensafely-2.1.2/opensafely/VERSION

@@ -0,0 +1 @@
+v2.1.2

opensafely-2.1.2/opensafely/_vendor/pipeline/constants.py

@@ -0,0 +1,19 @@
+# The magic action name which means "run every action"
+RUN_ALL_COMMAND = "run_all"
+
+LEVEL4_FILE_TYPES = {
+    # tables
+    ".csv",
+    # images
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".svg",
+    ".svgz",
+    # reports
+    ".html",
+    ".txt",
+    ".log",
+    ".json",
+    ".md",
+}

{opensafely-2.1.0 → opensafely-2.1.2}/opensafely/_vendor/pipeline/features.py

@@ -15,6 +15,8 @@ FEATURE_FLAGS_BY_VERSION = {
 
 LATEST_VERSION = max([v[0] for v in FEATURE_FLAGS_BY_VERSION.values()])
 
+MINIMUM_VERSION = 4
+
 
 def get_feature_flags_for_version(version: float) -> SimpleNamespace:
     if version > LATEST_VERSION:

{opensafely-2.1.0 → opensafely-2.1.2}/opensafely/_vendor/pipeline/loading.py

@@ -80,7 +80,7 @@ def parse_yaml_file(data: str | Path, filename: str | None = None) -> dict[str,
     if FAST_PARSER is not None:  # pragma: no cover
         try:
             return FAST_PARSER.load(data)  # type: ignore[no-any-return]
-        except Exception:
+        except Exception:  # noqa: S110, BLE001
             pass
 
     try:

{opensafely-2.1.0 → opensafely-2.1.2}/opensafely/_vendor/pipeline/models.py

@@ -4,16 +4,15 @@ import pathlib
 import re
 import shlex
 import warnings
-from collections import defaultdict
 from dataclasses import dataclass
 from typing import Any
 
 from .constants import RUN_ALL_COMMAND
 from .exceptions import InvalidPatternError, ValidationError
-from .features import LATEST_VERSION, get_feature_flags_for_version
+from .features import LATEST_VERSION, MINIMUM_VERSION, get_feature_flags_for_version
 from .validation import (
     validate_action_config,
-    validate_cohortextractor_outputs,
+    validate_actions_config,
     validate_ehrql_outputs,
     validate_glob_pattern,
     validate_no_kwargs,
@@ -21,6 +20,7 @@ from .validation import (
     validate_not_latest_tag,
     validate_not_run_all_action,
     validate_type,
+    validate_unique_output_paths,
 )
 
 
@@ -28,7 +28,6 @@ from .validation import (
 DB_COMMANDS = {
     "ehrql": ("generate-dataset", "generate-measures"),
     "sqlrunner": "*",  # all commands are valid
-    "cohortextractor": ("generate_cohort", "generate_codelist_report"),
     "databuilder": ("generate-dataset",),
 }
 
@@ -54,26 +53,6 @@ def is_database_action(args: list[str]) -> bool:
     return args[1] in db_commands
 
 
-@dataclass(frozen=True)
-class Expectations:
-    population_size: int
-
-    @classmethod
-    def build(
-        cls,
-        population_size: Any = None,
-        **kwargs: Any,
-    ) -> Expectations:
-        validate_no_kwargs(kwargs, "project `expectations` section")
-        try:
-            population_size = int(population_size)
-        except (TypeError, ValueError):
-            raise ValidationError(
-                "Project expectations population size must be a number",
-            )
-        return cls(population_size)
-
-
 @dataclass(frozen=True)
 class Outputs:
     highly_sensitive: dict[str, str] | None
@@ -95,7 +74,7 @@ class Outputs:
             and minimally_sensitive is None
         ):
             raise ValidationError(
-                f"must specify at least one output of: {', '.join(['highly_sensitive', 'moderately_sensitive', 'minimally_sensitive'])}"
+                "must specify at least one output of: highly_sensitive, moderately_sensitive, minimally_sensitive"
             )
 
         validate_no_kwargs(kwargs, f"`outputs` section for action {action_id}")
@@ -210,10 +189,11 @@ class Action:
                 )
         action = cls(action_id, outputs, run, needs, config, dummy_data_file)
 
-        if re.match(r"cohortextractor:\S+ generate_cohort", run.raw):
-            validate_cohortextractor_outputs(action_id, action)
-        if re.match(r"(ehrql|databuilder):\S+ generate[-_]dataset", run.raw):
-            validate_ehrql_outputs(action_id, action)
+        if match := re.match(
+            r"(ehrql|databuilder):\S+ generate[-_](?P<ehrql_type>dataset|measures)",
+            run.raw,
+        ):
+            validate_ehrql_outputs(action_id, action, match.group("ehrql_type"))
 
         return action
 
@@ -249,17 +229,14 @@ class Action:
 class Pipeline:
     version: float
     actions: dict[str, Action]
-    expectations: Expectations | None
 
     @classmethod
     def build(
         cls,
         version: Any = None,
         actions: Any = None,
-        expectations: Any = None,
         **kwargs: Any,
     ) -> Pipeline:
-        validate_no_kwargs(kwargs, "project")
         if version is None:
             raise ValidationError(
                 f"Project file must have a `version` attribute specifying which "
@@ -271,84 +248,44 @@ class Pipeline:
             version = float(version)
         except (TypeError, ValueError):
             raise ValidationError(
-                f"`version` must be a number between 1 and {LATEST_VERSION}"
+                f"`version` must be a number between {MINIMUM_VERSION} and {LATEST_VERSION}"
             )
         else:
+            if version < MINIMUM_VERSION:
+                raise ValidationError(
+                    f"Your project file is using a deprecated version ({version}); update to at least version {MINIMUM_VERSION}"
+                )
             if version != LATEST_VERSION:
                 warnings.warn(
                     f"ProjectWarning: Your project file is using an old version ({version}); consider updating to version {LATEST_VERSION}",
                     stacklevel=2,
                 )
 
+        validate_no_kwargs(kwargs, "project")
+
         feat = get_feature_flags_for_version(version)
 
         validate_type(actions, dict, "Project `actions` section")
 
-        _actions = dict()
+        _actions = {}
         for action_id, action_config in actions.items():
             validate_not_run_all_action(action_id, feat)
             validate_action_config(action_id, action_config)
             _actions[action_id] = Action.build(action_id, **action_config)
         actions = _actions
 
-        if feat.REMOVE_SUPPORT_FOR_COHORT_EXTRACTOR:
-            for config in actions.values():
-                validate_not_cohort_extractor_action(config)
+        for config in actions.values():
+            validate_not_cohort_extractor_action(config)
 
         if feat.REMOVE_SUPPORT_FOR_LATEST_TAG:
             for config in actions.values():
                 validate_not_latest_tag(config)
 
-        seen: dict[Command, list[str]] = defaultdict(list)
-        for name, config in actions.items():
-            run = config.run
-            if run in seen:
-                raise ValidationError(
-                    f"Action {name} has the same 'run' command as other actions: {' ,'.join(seen[run])}"
-                )
-            seen[run].append(name)
-
-        if feat.UNIQUE_OUTPUT_PATH:
-            # find duplicate paths defined in the outputs section
-            seen_files = []
-            for config in actions.values():
-                for output in config.outputs.dict().values():
-                    for filename in output.values():
-                        if filename in seen_files:
-                            raise ValidationError(
-                                f"Output path {filename} is not unique"
-                            )
-
-                        seen_files.append(filename)
-
-        for a in actions.values():
-            for n in a.needs:
-                if n not in actions:
-                    raise ValidationError(
-                        f"Action `{a.action_id}` references an unknown action in its `needs` list: {n}"
-                    )
-
-        if feat.REMOVE_SUPPORT_FOR_COHORT_EXTRACTOR:
-            if expectations is not None:
-                raise ValidationError(
-                    "Project includes `expectations` section, which is not supported in this version. "
-                    "This section is only applicable to deprecated cohortextractor actions; you can safely remove it."
-                )
-        elif feat.EXPECTATIONS_POPULATION:
-            if expectations is None:
-                raise ValidationError("Project must include `expectations` section")
-        else:
-            expectations = {"population_size": 1000}
+        validate_actions_config(actions)
 
-        if expectations is not None:
-            validate_type(expectations, dict, "Project `expectations` section")
-            if "population_size" not in expectations:
-                raise ValidationError(
-                    "Project `expectations` section must include `population_size` section",
-                )
-            expectations = Expectations.build(**expectations)
+        validate_unique_output_paths(actions)
 
-        return cls(version, actions, expectations)
+        return cls(version, actions)
 
     @property
     def all_actions(self) -> list[str]:
@@ -360,7 +297,7 @@ class Pipeline:
         We use a list comprehension rather than set operators as previously so we preserve
         the original order.
         """
-        return [action for action in self.actions.keys() if action != RUN_ALL_COMMAND]
+        return [action for action in self.actions if action != RUN_ALL_COMMAND]
 
     @property
     def action_images(self) -> set[str]:

{opensafely-2.1.0 → opensafely-2.1.2}/opensafely/_vendor/pipeline/validation.py

@@ -3,17 +3,17 @@ from __future__ import annotations
 import fnmatch
 import posixpath
 import warnings
+from collections import defaultdict
 from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath
 from types import SimpleNamespace
 from typing import TYPE_CHECKING, Any
 
 from .constants import LEVEL4_FILE_TYPES
 from .exceptions import InvalidPatternError, ValidationError
-from .outputs import get_output_dirs
 
 
 if TYPE_CHECKING:  # pragma: no cover
-    from .models import Action
+    from .models import Action, Command
 
 
 def validate_type(val: Any, exp_type: type, loc: str, optional: bool = False) -> None:
@@ -59,11 +59,10 @@ def validate_glob_pattern(pattern: str, privacy_level: str) -> None:
             "output paths must have a file type extension at the end"
         )
 
-    if privacy_level == "moderately_sensitive":
-        if path.suffix not in LEVEL4_FILE_TYPES:
-            raise InvalidPatternError(
-                f"{path} is not an allowed file type for moderately_sensitive outputs"
-            )
+    if privacy_level == "moderately_sensitive" and path.suffix not in LEVEL4_FILE_TYPES:
+        raise InvalidPatternError(
+            f"{path} is not an allowed file type for moderately_sensitive outputs"
+        )
 
     # Check that the path is in normal form
     if posixpath.normpath(pattern) != pattern:
@@ -92,6 +91,29 @@ def validate_action_config(action_id: str, action_config: Any) -> None:
             )
 
 
+def validate_actions_config(actions: dict[str, Action]) -> None:
+    """
+    Validates that actions are consistent and valid with respect to other actions
+    - run commands are not duplicated
+    - Specified `needs` are valid actions
+    """
+    seen: dict[Command, list[str]] = defaultdict(list)
+    for name, config in actions.items():
+        run = config.run
+        if run in seen:
+            raise ValidationError(
+                f"Action {name} has the same 'run' command as other actions: {' ,'.join(seen[run])}"
+            )
+        seen[run].append(name)
+
+    for a in actions.values():
+        for n in a.needs:
+            if n not in actions:
+                raise ValidationError(
+                    f"Action `{a.action_id}` references an unknown action in its `needs` list: {n}"
+                )
+
+
 def validate_not_cohort_extractor_action(action: Action) -> None:
     if action.run.parts[0].startswith("cohortextractor"):
         raise ValidationError(
@@ -121,49 +143,30 @@ def validate_not_latest_tag(action: Action) -> None:
         )
 
 
-def validate_cohortextractor_outputs(action_id: str, action: Action) -> None:
+def validate_unique_output_paths(actions: dict[str, Action]) -> None:
     """
-    Check cohortextractor's output config is valid for this command
-
-    We can't validate outputs in the Action or Outputs models because we need
-    to look up other fields (eg run).
+    Validates that paths defined in the outputs section are unique
     """
-    # ensure we only have output level defined.
-    num_output_levels = len(action.outputs)
-    if num_output_levels != 1:
-        raise ValidationError(
-            "A `generate_cohort` action must have exactly one output; "
-            f"{action_id} had {num_output_levels}"
-        )
+    seen_files = []
+    for config in actions.values():
+        for output in config.outputs.dict().values():
+            for filename in output.values():
+                if filename in seen_files:
+                    raise ValidationError(f"Output path {filename} is not unique")
 
-    output_dirs = get_output_dirs(action.outputs)
-    if len(output_dirs) == 1:
-        return
-
-    # If we detect multiple output directories but the command explicitly
-    # specifies an output directory then we assume the user knows what
-    # they're doing and don't attempt to modify the output directory or
-    # throw an error
-    flag = "--output-dir"
-    has_output_dir = any(
-        arg == flag or arg.startswith(f"{flag}=") for arg in action.run.parts
-    )
-    if not has_output_dir:
-        raise ValidationError(
-            f"generate_cohort command should produce output in only one "
-            f"directory, found {len(output_dirs)}:\n"
-            + "\n".join([f" - {d}/" for d in output_dirs])
-        )
+                seen_files.append(filename)
 
 
-def validate_ehrql_outputs(action_id: str, action: Action) -> None:
+def validate_ehrql_outputs(action_id: str, action: Action, ehrql_type: str) -> None:
     """
     Check ehrQL's output config is valid for this command
 
     We can't validate outputs in the Action or Outputs models because we need
     to look up other fields (eg run).
     """
-    if action.outputs.moderately_sensitive or action.outputs.minimally_sensitive:
+    if ehrql_type == "dataset" and (
+        action.outputs.moderately_sensitive or action.outputs.minimally_sensitive
+    ):
         raise ValidationError(
             f"`{action_id}` action uses `generate-dataset` and so all outputs must "
             f"be labelled `highly_sensitive`"
@@ -173,11 +176,18 @@ def validate_ehrql_outputs(action_id: str, action: Action) -> None:
     if not output_spec:
         raise ValidationError(
             f"`{action_id}` action does not provide an `--output` argument specifying "
-            f"where the results of `generate-dataset` should be stored"
+            f"where the results of `generate-{ehrql_type}` should be stored"
+        )
+
+    if ehrql_type == "dataset":
+        output_patterns = list((action.outputs.highly_sensitive or {}).values())
+    else:
+        assert ehrql_type == "measures"
+        output_patterns = list((action.outputs.highly_sensitive or {}).values()) + list(
+            (action.outputs.moderately_sensitive or {}).values()
         )
 
-    output_patterns = (action.outputs.highly_sensitive or {}).values()
-    if not output_patterns_match_spec(output_spec, list(output_patterns)):
+    if not output_patterns_match_spec(output_spec, output_patterns):
         expected = expected_pattern_for_spec(output_spec)
         formatted_patterns = "                   \n".join(output_patterns)
         raise ValidationError(

{opensafely-2.1.0 → opensafely-2.1.2}/opensafely.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensafely
-Version: 2.1.0
+Version: 2.1.2
 Summary: Command line tool for running OpenSAFELY studies locally.
 Author-email: OpenSAFELY <tech@opensafely.org>
 License: GPL-3.0

opensafely-2.1.0/opensafely/VERSION

@@ -1 +0,0 @@
-v2.1.0

opensafely-2.1.0/opensafely/_vendor/pipeline/constants.py

@@ -1,21 +0,0 @@
-# The magic action name which means "run every action"
-RUN_ALL_COMMAND = "run_all"
-
-LEVEL4_FILE_TYPES = set(
-    [
-        # tables
-        ".csv",
-        # images
-        ".jpg",
-        ".jpeg",
-        ".png",
-        ".svg",
-        ".svgz",
-        # reports
-        ".html",
-        ".txt",
-        ".log",
-        ".json",
-        ".md",
-    ]
-)