orchestrator-core 4.4.1__py3-none-any.whl → 4.5.0a3__py3-none-any.whl

orchestrator/search/retrieval/retrievers/semantic.py

@@ -0,0 +1,94 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sqlalchemy import Select, and_, cast, func, literal, or_, select
+from sqlalchemy.sql.expression import ColumnElement
+
+from orchestrator.db.models import AiSearchIndex
+from orchestrator.search.core.types import SearchMetadata
+
+from ..pagination import PaginationParams
+from .base import Retriever
+
+
+class SemanticRetriever(Retriever):
+    """Ranks results based on the minimum semantic vector distance."""
+
+    def __init__(self, vector_query: list[float], pagination_params: PaginationParams) -> None:
+        self.vector_query = vector_query
+        self.page_after_score = pagination_params.page_after_score
+        self.page_after_id = pagination_params.page_after_id
+
+    def apply(self, candidate_query: Select) -> Select:
+        cand = candidate_query.subquery()
+
+        dist = AiSearchIndex.embedding.l2_distance(self.vector_query)
+
+        raw_min = func.min(dist).over(partition_by=AiSearchIndex.entity_id)
+
+        # Normalize score to preserve ordering in accordance with other retrievers:
+        # smaller distance = higher score
+        similarity = literal(1.0, type_=self.SCORE_NUMERIC_TYPE) / (
+            literal(1.0, type_=self.SCORE_NUMERIC_TYPE) + cast(raw_min, self.SCORE_NUMERIC_TYPE)
+        )
+
+        score = cast(
+            func.round(cast(similarity, self.SCORE_NUMERIC_TYPE), self.SCORE_PRECISION), self.SCORE_NUMERIC_TYPE
+        ).label(self.SCORE_LABEL)
+
+        combined_query = (
+            select(
+                AiSearchIndex.entity_id,
+                score,
+                func.first_value(AiSearchIndex.value)
+                .over(partition_by=AiSearchIndex.entity_id, order_by=[dist.asc(), AiSearchIndex.path.asc()])
+                .label(self.HIGHLIGHT_TEXT_LABEL),
+                func.first_value(AiSearchIndex.path)
+                .over(partition_by=AiSearchIndex.entity_id, order_by=[dist.asc(), AiSearchIndex.path.asc()])
+                .label(self.HIGHLIGHT_PATH_LABEL),
+            )
+            .select_from(AiSearchIndex)
+            .join(cand, cand.c.entity_id == AiSearchIndex.entity_id)
+            .where(AiSearchIndex.embedding.isnot(None))
+            .distinct(AiSearchIndex.entity_id)
+        )
+        final_query = combined_query.subquery("ranked_semantic")
+
+        stmt = select(
+            final_query.c.entity_id,
+            final_query.c.score,
+            final_query.c.highlight_text,
+            final_query.c.highlight_path,
+        ).select_from(final_query)
+
+        stmt = self._apply_semantic_pagination(stmt, final_query.c.score, final_query.c.entity_id)
+
+        return stmt.order_by(final_query.c.score.desc().nulls_last(), final_query.c.entity_id.asc())
+
+    @property
+    def metadata(self) -> SearchMetadata:
+        return SearchMetadata.semantic()
+
+    def _apply_semantic_pagination(
+        self, stmt: Select, score_column: ColumnElement, entity_id_column: ColumnElement
+    ) -> Select:
+        """Apply semantic score pagination with precise Decimal handling."""
+        if self.page_after_score is not None and self.page_after_id is not None:
+            score_param = self._quantize_score_for_pagination(self.page_after_score)
+            stmt = stmt.where(
+                or_(
+                    score_column < score_param,
+                    and_(score_column == score_param, entity_id_column > self.page_after_id),
+                )
+            )
+        return stmt

orchestrator/search/retrieval/retrievers/structured.py

@@ -0,0 +1,39 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sqlalchemy import Select, literal, select
+
+from orchestrator.search.core.types import SearchMetadata
+
+from ..pagination import PaginationParams
+from .base import Retriever
+
+
+class StructuredRetriever(Retriever):
+    """Applies a dummy score for purely structured searches with no text query."""
+
+    def __init__(self, pagination_params: PaginationParams) -> None:
+        self.page_after_id = pagination_params.page_after_id
+
+    def apply(self, candidate_query: Select) -> Select:
+        cand = candidate_query.subquery()
+        stmt = select(cand.c.entity_id, literal(1.0).label("score")).select_from(cand)
+
+        if self.page_after_id:
+            stmt = stmt.where(cand.c.entity_id > self.page_after_id)
+
+        return stmt.order_by(cand.c.entity_id.asc())
+
+    @property
+    def metadata(self) -> SearchMetadata:
+        return SearchMetadata.structured()

orchestrator/search/retrieval/utils.py

@@ -0,0 +1,120 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import re
+
+import structlog
+from sqlalchemy import and_
+from sqlalchemy_utils.types.ltree import Ltree
+
+from orchestrator.db.database import WrappedSession
+from orchestrator.db.models import AiSearchIndex
+from orchestrator.search.core.types import EntityType
+from orchestrator.search.indexing.registry import ENTITY_CONFIG_REGISTRY
+from orchestrator.search.schemas.parameters import BaseSearchParameters
+from orchestrator.search.schemas.results import SearchResult
+
+logger = structlog.get_logger(__name__)
+
+
+def generate_highlight_indices(text: str, term: str) -> list[tuple[int, int]]:
+    """Finds all occurrences of individual words from the term, including both word boundary and substring matches."""
+    if not text or not term:
+        return []
+
+    all_matches = []
+    words = [w.strip() for w in term.split() if w.strip()]
+
+    for word in words:
+        # First find word boundary matches
+        word_boundary_pattern = rf"\b{re.escape(word)}\b"
+        word_matches = list(re.finditer(word_boundary_pattern, text, re.IGNORECASE))
+        all_matches.extend([(m.start(), m.end()) for m in word_matches])
+
+        # Then find all substring matches
+        substring_pattern = re.escape(word)
+        substring_matches = list(re.finditer(substring_pattern, text, re.IGNORECASE))
+        all_matches.extend([(m.start(), m.end()) for m in substring_matches])
+
+    return sorted(set(all_matches))
+
+
+def display_filtered_paths_only(
+    results: list[SearchResult], search_params: BaseSearchParameters, db_session: WrappedSession
+) -> None:
+    """Display only the paths that were searched for in the results."""
+    if not results:
+        logger.info("No results found.")
+        return
+
+    logger.info("--- Search Results ---")
+
+    searched_paths = search_params.filters.get_all_paths() if search_params.filters else []
+    if not searched_paths:
+        return
+
+    for result in results:
+        for path in searched_paths:
+            record: AiSearchIndex | None = (
+                db_session.query(AiSearchIndex)
+                .filter(and_(AiSearchIndex.entity_id == result.entity_id, AiSearchIndex.path == Ltree(path)))
+                .first()
+            )
+
+            if record:
+                logger.info(f"  {record.path}: {record.value}")
+
+        logger.info("-" * 40)
+
+
+def display_results(
+    results: list[SearchResult],
+    db_session: WrappedSession,
+    score_label: str = "Score",
+) -> None:
+    """Display search results, showing matched field when available or uuid+name for vector search."""
+    if not results:
+        logger.info("No results found.")
+        return
+
+    logger.info("--- Search Results ---")
+    for result in results:
+        entity_id = result.entity_id
+        score = result.score
+
+        # If we have a matching field from fuzzy search, display only that
+        if result.matching_field:
+            logger.info(f"Entity ID: {entity_id}")
+            logger.info(f"Matched field ({result.matching_field.path}): {result.matching_field.text}")
+            logger.info(f"{score_label}: {score:.4f}\n" + "-" * 20)
+            continue
+
+        index_records = db_session.query(AiSearchIndex).filter(AiSearchIndex.entity_id == entity_id).all()
+        if not index_records:
+            logger.warning(f"Could not find indexed records for entity_id={entity_id}")
+            continue
+
+        first_record = index_records[0]
+        kind = EntityType(first_record.entity_type)
+        config = ENTITY_CONFIG_REGISTRY[kind]
+
+        db_entity = db_session.get(config.table, entity_id) if config.table else None
+
+        if db_entity and config.traverser:
+            fields = config.traverser.get_fields(db_entity, config.pk_name, config.root_name)
+            result_obj = {p: v for p, v, _ in fields}
+            logger.info(json.dumps(result_obj, indent=2, default=str))
+            logger.info(f"{score_label}: {score:.4f}\n" + "-" * 20)
+        else:
+            logger.warning(f"Could not display entity {kind.value} with id={entity_id}")

orchestrator/search/retrieval/validation.py

@@ -0,0 +1,152 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sqlalchemy import select, text
+from sqlalchemy.exc import ProgrammingError
+from sqlalchemy_utils import Ltree
+
+from orchestrator.db import db
+from orchestrator.db.database import WrappedSession
+from orchestrator.db.models import AiSearchIndex
+from orchestrator.search.core.types import EntityType, FieldType
+from orchestrator.search.filters import FilterCondition, FilterTree, LtreeFilter, PathFilter
+from orchestrator.search.filters.definitions import operators_for
+from orchestrator.search.retrieval.exceptions import (
+    EmptyFilterPathError,
+    IncompatibleFilterTypeError,
+    InvalidEntityPrefixError,
+    InvalidLtreePatternError,
+    PathNotFoundError,
+)
+
+
+def is_filter_compatible_with_field_type(filter_condition: FilterCondition, field_type: FieldType) -> bool:
+    """Check whether a filter condition is compatible with a given field type.
+
+    Args:
+        filter_condition (FilterCondition): The filter condition instance to check.
+        field_type (FieldType): The type of field from the index schema.
+
+    Returns:
+        bool: True if the filter condition is valid for the given field type, False otherwise.
+    """
+
+    # LtreeFilter is for path filtering only and is thus compatible with all field types.
+    if isinstance(filter_condition, LtreeFilter):
+        return True
+
+    # Get valid operators for this field type and check if the filter's operator is valid.
+    valid_operators = operators_for(field_type)
+    return filter_condition.op in valid_operators
+
+
+def is_lquery_syntactically_valid(pattern: str, db_session: WrappedSession) -> bool:
+    """Validate whether a string is a syntactically correct `lquery` pattern.
+
+    Args:
+        pattern (str): The LTree lquery pattern string to validate.
+        db_session (WrappedSession): The database session used to test casting.
+
+    Returns:
+        bool: True if the pattern is valid, False if it fails to cast in PostgreSQL.
+    """
+
+    try:
+        with db_session.begin_nested():
+            db_session.execute(text("SELECT CAST(:pattern AS lquery)"), {"pattern": pattern})
+        return True
+    except ProgrammingError:
+        return False
+
+
+def get_structured_filter_schema() -> dict[str, str]:
+    """Retrieve all distinct filterable paths and their field types from the index.
+
+    Returns:
+        Dict[str, str]: Mapping of path strings to their corresponding field type values.
+    """
+
+    stmt = select(AiSearchIndex.path, AiSearchIndex.value_type).distinct().order_by(AiSearchIndex.path)
+    result = db.session.execute(stmt)
+    return {str(path): value_type.value for path, value_type in result}
+
+
+def validate_filter_path(path: str) -> str | None:
+    """Check if a given path exists in the index and return its field type.
+
+    Args:
+        path (str): The fully qualified LTree path.
+
+    Returns:
+        Optional[str]: The value type of the field if found, otherwise None.
+    """
+
+    stmt = select(AiSearchIndex.value_type).where(AiSearchIndex.path == Ltree(path)).limit(1)
+    result = db.session.execute(stmt).scalar_one_or_none()
+    return result.value if result else None
+
+
+async def complete_filter_validation(filter: PathFilter, entity_type: EntityType) -> None:
+    """Validate a PathFilter against the database schema and entity type.
+
+    Checks performed:
+    1. LTree filter syntax (for LtreeFilter only)
+    2. Non-empty path
+    3. Path exists in the database schema
+    4. Filter type matches the field's value_type
+    5. Path starts with the correct entity type prefix (unless wildcard)
+
+    Args:
+        filter (PathFilter): The filter to validate.
+        entity_type (EntityType): The entity type being searched.
+
+    Raises:
+        ValueError: If any of the validation checks fail.
+    """
+
+    # Ltree is a special case
+    if isinstance(filter.condition, LtreeFilter):
+        lquery_pattern = filter.condition.value
+        if not is_lquery_syntactically_valid(lquery_pattern, db.session):
+            raise InvalidLtreePatternError(lquery_pattern)
+        return
+
+    if not filter.path or not filter.path.strip():
+        raise EmptyFilterPathError()
+
+    # 1. Check if path exists in database
+    db_field_type_str = validate_filter_path(filter.path)
+    if db_field_type_str is None:
+        raise PathNotFoundError(filter.path)
+
+    db_field_type = FieldType(db_field_type_str)
+
+    # 2. Check filter compatibility with field type
+    if not is_filter_compatible_with_field_type(filter.condition, db_field_type):
+        expected_operators = operators_for(db_field_type)
+        raise IncompatibleFilterTypeError(
+            filter.condition.op.value, db_field_type.value, filter.path, expected_operators
+        )
+
+    # 3. Check entity type prefix requirements (unless it's a wildcard path)
+    expected_prefix = f"{entity_type.value.lower()}."
+    if not filter.path.startswith(expected_prefix) and not filter.path.startswith("*"):
+        raise InvalidEntityPrefixError(filter.path, expected_prefix, entity_type.value)
+
+
+async def validate_filter_tree(filters: FilterTree | None, entity_type: EntityType) -> None:
+    """Validate all PathFilter leaves in a FilterTree."""
+    if filters is None:
+        return
+    for leaf in filters.get_all_leaves():
+        await complete_filter_validation(leaf, entity_type)

orchestrator/search/schemas/__init__.py

@@ -0,0 +1,12 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

orchestrator/search/schemas/parameters.py

@@ -0,0 +1,129 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from orchestrator.search.core.types import ActionType, EntityType
+from orchestrator.search.filters import FilterTree
+
+
+class BaseSearchParameters(BaseModel):
+    """Base model with common search parameters."""
+
+    action: ActionType = Field(default=ActionType.SELECT, description="The action to perform.")
+    entity_type: EntityType
+
+    filters: FilterTree | None = Field(default=None, description="A list of structured filters to apply to the search.")
+
+    query: str | None = Field(
+        default=None, description="Unified search query - will be processed into vector_query and/or fuzzy_term"
+    )
+
+    limit: int = Field(default=10, ge=1, le=30, description="Maximum number of search results to return.")
+    model_config = ConfigDict(extra="forbid")
+
+    @classmethod
+    def create(cls, entity_type: EntityType, **kwargs: Any) -> "BaseSearchParameters":
+        try:
+            return PARAMETER_REGISTRY[entity_type](entity_type=entity_type, **kwargs)
+        except KeyError:
+            raise ValueError(f"No search parameter class found for entity type: {entity_type.value}")
+
+    @property
+    def vector_query(self) -> str | None:
+        """Extract vector query from unified query field."""
+        if not self.query:
+            return None
+        try:
+            uuid.UUID(self.query)
+            return None  # It's a UUID, so disable vector search.
+        except ValueError:
+            return self.query
+
+    @property
+    def fuzzy_term(self) -> str | None:
+        """Extract fuzzy term from unified query field."""
+        if not self.query:
+            return None
+
+        words = self.query.split()
+        # Only use fuzzy for single words
+        # otherwise, trigram operator filters out too much.
+        return self.query if len(words) == 1 else None
+
+
+class SubscriptionSearchParameters(BaseSearchParameters):
+    entity_type: Literal[EntityType.SUBSCRIPTION] = Field(
+        default=EntityType.SUBSCRIPTION, description="The type of entity to search."
+    )
+    model_config = ConfigDict(
+        json_schema_extra={
+            "title": "SearchSubscriptions",
+            "description": "Search subscriptions based on specific criteria.",
+            "examples": [
+                {
+                    "filters": {
+                        "op": "AND",
+                        "children": [
+                            {"path": "subscription.status", "condition": {"op": "eq", "value": "provisioning"}},
+                            {"path": "subscription.end_date", "condition": {"op": "gte", "value": "2025-01-01"}},
+                        ],
+                    }
+                }
+            ],
+        }
+    )
+
+
+class ProductSearchParameters(BaseSearchParameters):
+    entity_type: Literal[EntityType.PRODUCT] = Field(
+        default=EntityType.PRODUCT, description="The type of entity to search."
+    )
+    model_config = ConfigDict(
+        json_schema_extra={
+            "title": "SearchProducts",
+            "description": "Search products based on specific criteria.",
+            "examples": [
+                {
+                    "filters": [
+                        {"path": "product.product_type", "condition": {"op": "eq", "value": "Shop"}},
+                    ]
+                }
+            ],
+        }
+    )
+
+
+class WorkflowSearchParameters(BaseSearchParameters):
+    entity_type: Literal[EntityType.WORKFLOW] = Field(
+        default=EntityType.WORKFLOW, description="The type of entity to search."
+    )
+
+
+class ProcessSearchParameters(BaseSearchParameters):
+    """Search parameters specifically for PROCESS entities."""
+
+    entity_type: Literal[EntityType.PROCESS] = Field(
+        default=EntityType.PROCESS, description="The type of entity to search."
+    )
+
+
+PARAMETER_REGISTRY: dict[EntityType, type[BaseSearchParameters]] = {
+    EntityType.SUBSCRIPTION: SubscriptionSearchParameters,
+    EntityType.PRODUCT: ProductSearchParameters,
+    EntityType.WORKFLOW: WorkflowSearchParameters,
+    EntityType.PROCESS: ProcessSearchParameters,
+}

orchestrator/search/schemas/results.py

@@ -0,0 +1,77 @@
+# Copyright 2019-2025 SURF, GÉANT.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from orchestrator.search.core.types import FilterOp, SearchMetadata, UIType
+
+
+class MatchingField(BaseModel):
+    """Contains the field that contributed most to the (fuzzy) search result."""
+
+    text: str
+    path: str
+    highlight_indices: list[tuple[int, int]] | None = None
+
+
+class SearchResult(BaseModel):
+    """Represents a single search result item."""
+
+    entity_id: str
+    score: float
+    perfect_match: int = 0
+    matching_field: MatchingField | None = None
+
+
+class SearchResponse(BaseModel):
+    """Response containing search results and metadata."""
+
+    results: list[SearchResult]
+    metadata: SearchMetadata
+
+
+class ValueSchema(BaseModel):
+    kind: UIType | Literal["none", "object"] = UIType.STRING
+    fields: dict[str, "ValueSchema"] | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LeafInfo(BaseModel):
+    name: str
+    ui_types: list[UIType]
+    paths: list[str]
+
+    model_config = ConfigDict(
+        extra="forbid",
+        use_enum_values=True,
+    )
+
+
+class ComponentInfo(BaseModel):
+    name: str
+    ui_types: list[UIType]
+
+    model_config = ConfigDict(
+        extra="forbid",
+        use_enum_values=True,
+    )
+
+
+class TypeDefinition(BaseModel):
+    operators: list[FilterOp]
+    value_schema: dict[FilterOp, ValueSchema]
+
+    model_config = ConfigDict(use_enum_values=True)

orchestrator/services/settings_env_variables.py

@@ -14,7 +14,7 @@
 from typing import Any, Dict, Type
 
 from pydantic import SecretStr as PydanticSecretStr
-from pydantic_core import MultiHostUrl, Url
+from pydantic.networks import AnyUrl, _BaseMultiHostUrl
 from pydantic_settings import BaseSettings
 
 from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
@@ -34,7 +34,7 @@ def mask_value(key: str, value: Any) -> Any:
     key_lower = key.lower()
     is_sensitive_key = "secret" in key_lower or "password" in key_lower
 
-    if is_sensitive_key or isinstance(value, (OrchSecretStr, PydanticSecretStr, MultiHostUrl, Url)):
+    if is_sensitive_key or isinstance(value, (OrchSecretStr, PydanticSecretStr, _BaseMultiHostUrl, AnyUrl)):
         return MASK
 
     return value

orchestrator/settings.py

@@ -1,4 +1,4 @@
-# Copyright 2019-2020 SURF, GÉANT.
+# Copyright 2019-2025 SURF, GÉANT.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

orchestrator/workflows/tasks/validate_products.py

@@ -105,7 +105,7 @@ def check_that_products_have_create_modify_and_terminate_workflows() -> State:
     product_data = get_products(filters=[ProductTable.status == "active"])
 
     workflows_not_complete: list = []
-    targets = ["CREATE", "TERMINATE", "MODIFY", "VALIDATE"]
+    targets = ["CREATE", "TERMINATE", "MODIFY", "RECONCILE", "VALIDATE"]
     for product in product_data:
         workflows = {c.target for c in product.workflows if c.target in targets and c.name != "modify_note"}
         if len(workflows) < len(targets):