qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/change_owners/change_types.py

@@ -16,14 +16,13 @@ from enum import Enum
 from typing import (
     Any,
     Optional,
-    Tuple,
 )
 
-import anymarkup
 import jinja2
 import jinja2.meta
 import jsonpath_ng
 import networkx
+from pydantic import Json
 
 from reconcile.change_owners.approver import (
     Approver,
@@ -34,13 +33,7 @@ from reconcile.change_owners.bundle import (
     FileDiffResolver,
     FileRef,
 )
-from reconcile.change_owners.diff import (
-    SHA256SUM_FIELD_NAME,
-    SHA256SUM_PATH,
-    Diff,
-    DiffType,
-    extract_diffs,
-)
+from reconcile.change_owners.diff import Diff
 from reconcile.gql_definitions.change_owners.queries.change_types import (
     ChangeTypeChangeDetectorChangeTypeProviderV1,
     ChangeTypeChangeDetectorJsonPathProviderV1,
@@ -62,11 +55,12 @@ class ChangeTypePriority(Enum):
     CRITICAL = "critical"
     URGENT = "urgent"
     HIGH = "high"
+    PROGRESSIVE_DELIVERY = "progressive-delivery"
     MEDIUM = "medium"
     LOW = "low"
 
 
-def parent_of_jsonpath(path: jsonpath_ng.JSONPath) -> Optional[jsonpath_ng.JSONPath]:
+def parent_of_jsonpath(path: jsonpath_ng.JSONPath) -> jsonpath_ng.JSONPath | None:
     # todo - figure out if this is enough of if we have other
     # structures where a parent can be extracted
     if isinstance(path, jsonpath_ng.Child):
@@ -86,6 +80,20 @@ class DiffCoverage:
 
     parent: Optional["DiffCoverage"] = None
 
+    @property
+    def diff_fragments(self) -> Sequence["DiffCoverage"]:
+        return self._split_into
+
+    @property
+    def change_owner_labels(self) -> set[str]:
+        """
+        Returns a list of change-owner labels of all involved change-type contexts.
+        """
+        labels = {label for c in self.coverage for label in c.change_owner_labels or {}}
+        for _split in self._split_into:
+            labels.update(_split.change_owner_labels)
+        return labels
+
     def relative_path(self) -> jsonpath_ng.JSONPath:
         if self.parent:
             path = remove_prefix_from_path(self.diff.path, self.parent.diff.path)
@@ -192,7 +200,7 @@ class DiffCoverage:
 
         return None
 
-    def add_covering_context(self, ctx: "ChangeTypeContext"):
+    def add_covering_context(self, ctx: "ChangeTypeContext") -> None:
         self.coverage.append(ctx)
         for s in self._split_into:
             s.add_covering_context(ctx)
@@ -216,172 +224,6 @@ class DiffCoverage:
         return coverages
 
 
-@dataclass
-class BundleFileChange:
-    """
-    Represents a file within an app-interface bundle that changed during an MR.
-    It holds the old and new state of that file, along with precise differences
-    between those states.
-    """
-
-    fileref: FileRef
-    old: Optional[dict[str, Any]]
-    new: Optional[dict[str, Any]]
-    diffs: list[Diff]
-    _diff_coverage: dict[str, DiffCoverage] = field(init=False, default_factory=dict)
-
-    def __post_init__(self) -> None:
-        self._diff_coverage = {d.path_str(): DiffCoverage(d, []) for d in self.diffs}
-
-    def cover_changes(self, change_type_context: "ChangeTypeContext") -> list[Diff]:
-        """
-        Figure out if a ChangeTypeV1 covers detected changes within the BundleFile.
-        Base idea:
-
-        - a ChangeTypeV1 defines path patterns that are considered self-approvable
-        - if a change (diff) is located under one of the allowed paths of the
-          ChangeTypeV1, it is considered "covered" by that ChangeTypeV1 in a certain
-          context (e.g. a RoleV1) and allows the approvers of that context (e.g.
-          the members of that role) to approve that particular change.
-
-        The change-type contexts that cover a change, are registered in the
-        `DiffCoverage.coverage` list right next to the Diff that is covered.
-        """
-        covered_diffs = {}
-        # observe the new state for added fields or list items or entire object sutrees
-        covered_diffs.update(
-            self._cover_changes_for_diffs(
-                self._filter_diffs([DiffType.ADDED, DiffType.CHANGED]),
-                self.new,
-                change_type_context,
-            )
-        )
-        # look at the old state for removed fields or list items or object subtrees
-        covered_diffs.update(
-            self._cover_changes_for_diffs(
-                self._filter_diffs([DiffType.REMOVED]), self.old, change_type_context
-            )
-        )
-        return list(covered_diffs.values())
-
-    def _cover_changes_for_diffs(
-        self,
-        diffs: list[DiffCoverage],
-        file_content: Any,
-        change_type_context: "ChangeTypeContext",
-    ) -> dict[str, Diff]:
-
-        covered_diffs = {}
-        if diffs:
-            for (
-                allowed_path
-            ) in change_type_context.change_type_processor.allowed_changed_paths(
-                self.fileref, file_content, change_type_context
-            ):
-                for dc in diffs:
-                    if dc.changed_path_covered_by_path(allowed_path):
-                        covered_diffs[dc.diff.path_str()] = dc.diff
-                        dc.coverage.append(change_type_context)
-                    elif SHA256SUM_PATH != allowed_path and dc.path_under_changed_path(
-                        allowed_path
-                    ):
-                        # the self-service path allowed by the change-type is covering
-                        # only parts of the diff. we will split the diff into a
-                        # smaller part, that can be covered by the change-type.
-                        # but the rest of the diff needs to be covered by another
-                        # change-type, either in full or again as a split.
-                        sub_dc = dc.split(allowed_path, change_type_context)
-                        if not sub_dc:
-                            raise Exception(
-                                f"unable to create a subdiff for path {allowed_path} on diff {dc.diff.path_str()}"
-                            )
-                        covered_diffs[str(allowed_path)] = sub_dc.diff
-
-        return covered_diffs
-
-    def _filter_diffs(self, diff_types: list[DiffType]) -> list[DiffCoverage]:
-        return [
-            d for d in self._diff_coverage.values() if d.diff.diff_type in diff_types
-        ]
-
-    def all_changes_covered(self) -> bool:
-        return all(d.is_covered() for d in self.diff_coverage)
-
-    def raw_diff_count(self) -> int:
-        return len(self._diff_coverage)
-
-    @property
-    def diff_coverage(self) -> Sequence[DiffCoverage]:
-        """
-        returns the meaningful set of diffs, potentially more fine grained than
-        what was originally detected for to the BundleFileChange.
-        """
-        coverages: list[DiffCoverage] = []
-        for dc in self._diff_coverage.values():
-            coverages.extend(dc.fine_grained_diff_coverages().values())
-        return coverages
-
-    def involved_change_types(self) -> list["ChangeTypeProcessor"]:
-        """
-        returns all the change-types that are involved in the coverage
-        of all changes
-        """
-        change_types = []
-        for dc in self.diff_coverage:
-            for ctx in dc.coverage:
-                change_types.append(ctx.change_type_processor)
-        return change_types
-
-
-def parse_resource_file_content(content: Optional[Any]) -> tuple[Any, Optional[str]]:
-    if content:
-        try:
-            data = anymarkup.parse(content, force_types=None)
-            return data, data.get("$schema")
-        except Exception:
-            # not parsable content - we will just deal with the plain content
-            return content, None
-    else:
-        return None, None
-
-
-def create_bundle_file_change(
-    path: str,
-    schema: Optional[str],
-    file_type: BundleFileType,
-    old_file_content: Any,
-    new_file_content: Any,
-) -> Optional[BundleFileChange]:
-    """
-    this is a factory method that creates a BundleFileChange object based
-    on the old and new content of a file from app-interface. it detects differences
-    within the old and new state of the file and represents them as instances
-    of the Diff dataclass. for diff detection, the amazing `deepdiff` python
-    library is used.
-    """
-    fileref = FileRef(path=path, schema=schema, file_type=file_type)
-
-    # try to parse the content of a resourcefile
-    # it falls back to the plain file content if parsing does not work
-    if file_type == BundleFileType.RESOURCEFILE:
-        old_file_content, _ = parse_resource_file_content(old_file_content)
-        new_file_content, _ = parse_resource_file_content(new_file_content)
-
-    diffs = extract_diffs(
-        old_file_content=old_file_content,
-        new_file_content=new_file_content,
-    )
-
-    if diffs:
-        return BundleFileChange(
-            fileref=fileref,
-            old=old_file_content,
-            new=new_file_content,
-            diffs=diffs,
-        )
-    return None
-
-
 class PathExpression:
     """
     PathExpression is a wrapper around a JSONPath expression that can contain
@@ -411,14 +253,12 @@ class PathExpression:
         if self.parsed_jsonpath:
             return self.parsed_jsonpath
 
-        expr = self.template.render(
-            {
-                self.CTX_FILE_PATH_VAR_NAME: ctx.context_file.path,
-            }
-        )
+        expr = self.template.render({
+            self.CTX_FILE_PATH_VAR_NAME: ctx.context_file.path,
+        })
         return parse_jsonpath(expr)
 
-    def __eq__(self, obj):
+    def __eq__(self, obj: object) -> bool:
         return (
             isinstance(obj, PathExpression)
             and obj.jsonpath_expression == self.jsonpath_expression
@@ -426,20 +266,35 @@ class PathExpression:
 
 
 @dataclass
-class OwnershipContext:
+class FileChange:
+    file_ref: FileRef
+    old: dict[str, Any] | None
+    new: dict[str, Any] | None
+    old_backrefs: set[FileRef] = field(default_factory=set)
+    new_backrefs: set[FileRef] = field(default_factory=set)
+
+
+class OwnershipContext(ABC):
+    @abstractmethod
+    def find_ownership_context(
+        self,
+        context_schema: str | None,
+        change: FileChange,
+    ) -> list[FileRef]: ...
+
+
+@dataclass
+class ForwardrefOwnershipContext(OwnershipContext):
     selector: jsonpath_ng.JSONPath
-    when: Optional[str]
+    when: str | None = None
 
     def find_ownership_context(
         self,
-        context_schema: Optional[str],
-        old_data: Optional[dict[str, Any]] = None,
-        new_data: Optional[dict[str, Any]] = None,
+        context_schema: str | None,
+        change: FileChange,
     ) -> list[FileRef]:
-
-        # extract contexts
-        old_contexts = {e.value for e in self.selector.find(old_data)}
-        new_contexts = {e.value for e in self.selector.find(new_data)}
+        old_contexts = {e.value for e in self.selector.find(change.old)}
+        new_contexts = {e.value for e in self.selector.find(change.new)}
 
         # apply conditions
         if self.when == "added":
@@ -461,6 +316,46 @@ class OwnershipContext:
         ]
 
 
+@dataclass
+class BackrefOwnershipContext(OwnershipContext):
+    selector: jsonpath_ng.JSONPath
+    file_diff_resolver: FileDiffResolver
+    when: str | None = None
+
+    def find_ownership_context(
+        self,
+        context_schema: str | None,
+        change: FileChange,
+    ) -> list[FileRef]:
+        # get backref datafile content
+        backref_datafile_content = {
+            ref: self.file_diff_resolver.lookup_file_diff(ref)
+            for ref in change.old_backrefs.union(change.new_backrefs)
+        }
+
+        # extract contexts
+        # we only care for those backrefs that mention the changed file at the selector
+        old_contexts = {
+            ref
+            for ref, data in backref_datafile_content.items()
+            if any(f.value == change.file_ref.path for f in self.selector.find(data[0]))
+        }
+        new_contexts = {
+            ref
+            for ref, data in backref_datafile_content.items()
+            if any(f.value == change.file_ref.path for f in self.selector.find(data[1]))
+        }
+
+        # apply conditions
+        if self.when == "added":
+            return list(new_contexts.difference(old_contexts))
+        if self.when == "removed":
+            return list(old_contexts.difference(new_contexts))
+        if self.when is None and old_contexts == new_contexts:
+            return list(old_contexts)
+        return []
+
+
 @dataclass
 class ContextExpansion:
     """
@@ -475,16 +370,22 @@ class ContextExpansion:
     def expand_from_file_ref(
         self,
         file_ref: FileRef,
-        expansion_trail: Set[Tuple[str, FileRef]],
+        expansion_trail: Set[tuple[str, FileRef]],
     ) -> list["ResolvedContext"]:
         old_data, new_data = self.file_diff_resolver.lookup_file_diff(file_ref)
-        return self.expand(old_data, new_data, expansion_trail)
+        return self.expand(
+            FileChange(
+                file_ref=file_ref,
+                old=old_data,
+                new=new_data,
+            ),
+            expansion_trail,
+        )
 
     def expand(
         self,
-        old_data: Optional[dict[str, Any]],
-        new_data: Optional[dict[str, Any]],
-        expansion_trail: Set[Tuple[str, FileRef]],
+        change: FileChange,
+        expansion_trail: Set[tuple[str, FileRef]],
     ) -> list["ResolvedContext"]:
         """
         Find context based on the `self.context`, lookup the file diff for
@@ -493,17 +394,18 @@ class ContextExpansion:
         """
         context_file_refs = self.context.find_ownership_context(
             context_schema=self.change_type.context_schema,
-            old_data=old_data,
-            new_data=new_data,
+            change=change,
         )
-        expaned_context_file_refs: list["ResolvedContext"] = []
+        expaned_context_file_refs: list[ResolvedContext] = []
         for ref in context_file_refs:
             ref_old_data, ref_new_data = self.file_diff_resolver.lookup_file_diff(ref)
             expaned_context_file_refs.extend(
                 self.change_type.find_context_file_refs(
-                    ref,
-                    old_data=ref_old_data,
-                    new_data=ref_new_data,
+                    change=FileChange(
+                        file_ref=ref,
+                        old=ref_old_data,
+                        new=ref_new_data,
+                    ),
                     expansion_trail=expansion_trail,
                 )
             )
@@ -528,28 +430,25 @@ class ChangeDetector(ABC):
     Represents an item from a change-types `change` list.
     """
 
-    context_schema: Optional[str]
-    change_schema: Optional[str]
-    context: Optional[OwnershipContext]
+    context_schema: str | None
+    change_schema: str | None
+    context: OwnershipContext | None
 
     @abstractmethod
     def find_context_file_refs(
         self,
-        old_data: Optional[dict[str, Any]] = None,
-        new_data: Optional[dict[str, Any]] = None,
-    ) -> list[FileRef]:
-        ...
+        change: FileChange,
+    ) -> list[FileRef]: ...
 
 
 @dataclass
 class JsonPathChangeDetector(ChangeDetector):
     json_path_selectors: list[str]
 
-    def __post_init__(self):
+    def __post_init__(self) -> None:
         self._json_path_expressions = [
             PathExpression(jsonpath_expression)
             for jsonpath_expression in self.json_path_selectors
-            + [f"'{SHA256SUM_FIELD_NAME}'"]
         ]
 
     @property
@@ -558,12 +457,12 @@ class JsonPathChangeDetector(ChangeDetector):
 
     def find_context_file_refs(
         self,
-        old_data: Optional[dict[str, Any]] = None,
-        new_data: Optional[dict[str, Any]] = None,
+        change: FileChange,
     ) -> list[FileRef]:
         if self.context:
             return self.context.find_ownership_context(
-                self.context_schema, old_data, new_data
+                context_schema=self.context_schema,
+                change=change,
             )
         return []
 
@@ -577,19 +476,22 @@ class ChangeTypeProcessor:
     """
 
     name: str
+    labels: Json | None
     description: str
     priority: ChangeTypePriority
     context_type: BundleFileType
-    context_schema: Optional[str]
+    context_schema: str | None
     disabled: bool
     implicit_ownership: list[ChangeTypeImplicitOwnershipV1]
+    restrictive: bool | None = False
 
-    def __post_init__(self):
+    def __post_init__(self) -> None:
         self._expressions_by_file_type_schema: dict[
-            tuple[BundleFileType, Optional[str]], list[PathExpression]
+            tuple[BundleFileType, str | None], list[PathExpression]
         ] = defaultdict(list)
         self._change_detectors: list[ChangeDetector] = []
         self._context_expansions: list[ContextExpansion] = []
+        self._heritage: set[str] = set()
 
     @property
     def change_detectors(self) -> Sequence[ChangeDetector]:
@@ -597,10 +499,8 @@ class ChangeTypeProcessor:
 
     def find_context_file_refs(
         self,
-        file_ref: FileRef,
-        old_data: Optional[dict[str, Any]],
-        new_data: Optional[dict[str, Any]],
-        expansion_trail: Set[Tuple[str, FileRef]],
+        change: FileChange,
+        expansion_trail: Set[tuple[str, FileRef]],
     ) -> list[ResolvedContext]:
         """
         ChangeTypeV1 are attached to bundle files, react to changes within
@@ -659,31 +559,31 @@ class ChangeTypeProcessor:
 
         # prevent infinite ownership resolution
         expansion_trail_copy = set(expansion_trail)
-        if (self.name, file_ref) in expansion_trail_copy:
+        if (self.name, change.file_ref) in expansion_trail_copy:
             return contexts
 
-        expansion_trail_copy.add((self.name, file_ref))
+        expansion_trail_copy.add((self.name, change.file_ref))
 
         # direct context extraction
         # the changed file itself is giving the context for approver extraction
         # see doc string for more details
-        if self.context_schema is None or self.context_schema == file_ref.schema:
+        if self.context_schema is None or self.context_schema == change.file_ref.schema:
             contexts.append(
                 ResolvedContext(
-                    owned_file_ref=file_ref,
-                    context_file_ref=file_ref,
+                    owned_file_ref=change.file_ref,
+                    context_file_ref=change.file_ref,
                     change_type=self,
                 )
             )
 
             # expand context based on change-type composition
             for ce in self._context_expansions:
-                for ec in ce.expand(old_data, new_data, expansion_trail_copy):
+                for ec in ce.expand(change, expansion_trail_copy):
                     # add expanded contexts (derived owned files)
                     contexts.append(
                         ResolvedContext(
                             owned_file_ref=ec.owned_file_ref,
-                            context_file_ref=file_ref,
+                            context_file_ref=change.file_ref,
                             change_type=ec.change_type,
                         )
                     )
@@ -693,8 +593,8 @@ class ChangeTypeProcessor:
         # file with a `context.selector`
         # see doc string for more details
         for c in self.change_detectors:
-            if c.change_schema == file_ref.schema:
-                for ctx_file_ref in c.find_context_file_refs(old_data, new_data):
+            if c.change_schema == change.file_ref.schema:
+                for ctx_file_ref in c.find_context_file_refs(change):
                     contexts.append(
                         ResolvedContext(
                             owned_file_ref=ctx_file_ref,
@@ -740,7 +640,7 @@ class ChangeTypeProcessor:
     def _allowed_changed_paths_for_file_type_and_schema(
         self,
         file_type: BundleFileType,
-        file_schema: Optional[str],
+        file_schema: str | None,
         file_content: Any,
         ctx: "ChangeTypeContext",
     ) -> list[jsonpath_ng.JSONPath]:
@@ -750,16 +650,14 @@ class ChangeTypeProcessor:
             file_schema,
         ) in self._expressions_by_file_type_schema:
             for change_type_path_expression in self._expressions_by_file_type_schema[
-                (file_type, file_schema)
+                file_type, file_schema
             ]:
-                paths.extend(
-                    [
-                        p.full_path
-                        for p in change_type_path_expression.jsonpath_for_context(
-                            ctx
-                        ).find(file_content)
-                    ]
-                )
+                paths.extend([
+                    p.full_path
+                    for p in change_type_path_expression.jsonpath_for_context(ctx).find(
+                        file_content
+                    )
+                ])
         return paths
 
     def add_change_detector(
@@ -770,7 +668,7 @@ class ChangeTypeProcessor:
             self._change_detectors.append(detector)
             change_schema = detector.change_schema or self.context_schema
             expressions = self._expressions_by_file_type_schema[
-                (self.context_type, change_schema)
+                self.context_type, change_schema
             ]
             for path_expression in detector.json_path_expressions:
                 if path_expression not in expressions:
@@ -780,9 +678,34 @@ class ChangeTypeProcessor:
                 f"{type(detector)} is not a supported change detection provider within ChangeTypes"
             )
 
-    def add_context_expansion(self, context_expansion: ContextExpansion):
+    def add_context_expansion(self, context_expansion: ContextExpansion) -> None:
         self._context_expansions.append(context_expansion)
 
+    def inherit_from(self, other: "ChangeTypeProcessor") -> None:
+        for detector in other.change_detectors:
+            self.add_change_detector(detector)
+        other._heritage = self.lineage.union(other._heritage)
+
+    @property
+    def lineage(self) -> set[str]:
+        return self._heritage.union({self.name})
+
+
+def build_ownership_context(
+    file_diff_resolver: FileDiffResolver,
+    selector: jsonpath_ng.JSONPath,
+    when: str | None = None,
+    where: str | None = None,
+) -> OwnershipContext:
+    """
+    create an OwnershipContext object based on the provided parameters
+    """
+    if where == "backrefs":
+        return BackrefOwnershipContext(
+            file_diff_resolver=file_diff_resolver, selector=selector, when=when
+        )
+    return ForwardrefOwnershipContext(selector=selector, when=when)
+
 
 def init_change_type_processors(
     change_types: Sequence[ChangeTypeV1], file_diff_resolver: FileDiffResolver
@@ -795,11 +718,13 @@ def init_change_type_processors(
         # build raw change-type-processor
         processors[change_type.name] = ChangeTypeProcessor(
             name=change_type.name,
+            labels=change_type.labels,
             description=change_type.description,
             priority=ChangeTypePriority(change_type.priority),
             context_type=BundleFileType[change_type.context_type.upper()],
             context_schema=change_type.context_schema,
             disabled=bool(change_type.disabled),
+            restrictive=bool(change_type.restrictive),
             implicit_ownership=change_type.implicit_ownership or [],
         )
         # register inheritance edges for cycle detection
@@ -814,9 +739,11 @@ def init_change_type_processors(
             if isinstance(change_detector, ChangeTypeChangeDetectorJsonPathProviderV1):
                 ownership_context = None
                 if change_detector.context:
-                    ownership_context = OwnershipContext(
+                    ownership_context = build_ownership_context(
+                        file_diff_resolver=file_diff_resolver,
                         selector=parse_jsonpath(change_detector.context.selector),
                         when=change_detector.context.when,
+                        where=change_detector.context.where,
                     )
                 processor.add_change_detector(
                     JsonPathChangeDetector(
@@ -840,11 +767,13 @@ def init_change_type_processors(
                     processors[ct.name].add_context_expansion(
                         ContextExpansion(
                             change_type=processor,
-                            context=OwnershipContext(
+                            context=build_ownership_context(
+                                file_diff_resolver=file_diff_resolver,
                                 selector=parse_jsonpath(
                                     change_detector.ownership_context.selector
                                 ),
                                 when=change_detector.ownership_context.when,
+                                where=change_detector.ownership_context.where,
                             ),
                             file_diff_resolver=file_diff_resolver,
                         )
@@ -875,8 +804,13 @@ def init_change_type_processors(
                     f"change-type '{ctp.name}' inherits from '{d}' "
                     "but has a different context_schema"
                 )
-            for detector in processors[d].change_detectors:
-                ctp.add_change_detector(detector)
+
+            # the higher level change type adopts the change detectors from his decentents
+            ctp.inherit_from(processors[d])
+
+            # the decentents adopt the context expansions from the higher level change type
+            for ce in ctp._context_expansions:
+                processors[d].add_context_expansion(ce)
 
     return processors
 
@@ -909,7 +843,8 @@ class ChangeTypeContext:
     origin: str
     context_file: FileRef
     approvers: list[Approver]
-    approver_reachability: Optional[list[ApproverReachability]] = None
+    approver_reachability: list[ApproverReachability] | None = None
+    change_owner_labels: set[str] | None = None
 
     @property
     def disabled(self) -> bool:
@@ -927,20 +862,3 @@ JSON_PATH_ROOT = "$"
 
 def change_path_covered_by_allowed_path(changed_path: str, allowed_path: str) -> bool:
     return changed_path.startswith(allowed_path) or allowed_path == JSON_PATH_ROOT
-
-
-def get_priority_for_changes(
-    bundle_file_changes: list[BundleFileChange],
-) -> Optional[ChangeTypePriority]:
-    """
-    Finds the lowest priority of all change types involved in the provided bundle file changes.
-    """
-    priorities: set[ChangeTypePriority] = set()
-    for bfc in bundle_file_changes:
-        for ct in bfc.involved_change_types():
-            priorities.add(ct.priority)
-    # get the lowest priority
-    for p in reversed(ChangeTypePriority):
-        if p in priorities:
-            return p
-    return None