pycarlo 0.10.210__py3-none-any.whl → 0.12.57__py3-none-any.whl

pycarlo/features/metadata/__init__.py

@@ -1,15 +1,32 @@
-from pycarlo.features.metadata.allow_block_list import (
-    AllowBlockList,
+from pycarlo.features.metadata.asset_allow_block_list import AssetAllowBlockList
+from pycarlo.features.metadata.asset_filters_container import AssetFiltersContainer
+from pycarlo.features.metadata.base_allow_block_list import (
+    BaseAllowBlockList,
+    ComparisonType,
     FilterEffectType,
+    FilterRule,
     FilterType,
+    RuleEffect,
+)
+from pycarlo.features.metadata.metadata_allow_block_list import (
+    MetadataAllowBlockList,
     MetadataFilter,
 )
 from pycarlo.features.metadata.metadata_filters_container import MetadataFiltersContainer
 
 __all__ = [
+    # Base classes
+    "FilterRule",
+    "BaseAllowBlockList",
     "FilterEffectType",
+    "RuleEffect",
     "FilterType",
+    "ComparisonType",
+    # Metadata filtering classes
     "MetadataFilter",
-    "AllowBlockList",
+    "MetadataAllowBlockList",
     "MetadataFiltersContainer",
+    # Asset filtering classes
+    "AssetAllowBlockList",
+    "AssetFiltersContainer",
 ]

pycarlo/features/metadata/asset_allow_block_list.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from dataclasses_json import DataClassJsonMixin
+
+from pycarlo.common import get_logger
+from pycarlo.features.metadata.base_allow_block_list import BaseAllowBlockList, FilterRule
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class AssetAllowBlockList(BaseAllowBlockList[FilterRule], DataClassJsonMixin):
+    # JSON deserialization fails without this ugly override
+    rules: Optional[List[FilterRule]] = field(default_factory=list)
+
+    asset_type: Optional[str] = None
+
+    def __post_init__(self):
+        # We can't remove the default value because of properties with defaults in the parent class.
+        if not self.asset_type:
+            raise ValueError("asset_type is required")

pycarlo/features/metadata/asset_filters_container.py

@@ -0,0 +1,79 @@
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+from dataclasses_json import DataClassJsonMixin
+
+from .asset_allow_block_list import AssetAllowBlockList
+from .base_allow_block_list import FilterEffectType
+
+# Mapping of resource types to their supported asset types for collection preferences.
+# This is used for validating asset collection preferences.
+# When support for filtering an asset type is implemented in the DC, it should be added here.
+# The reason it is here instead of in Monolith, is so that it can be referenced by the CLI.
+# The pycarlo version in CLI and monolith should be updated after updating this and releasing a
+# new version.
+ASSET_TYPE_ATTRIBUTES = {"tableau": {"project": ["name"], "workbook": ["name", "luid"]}}
+
+
+@dataclass
+class AssetFiltersContainer(DataClassJsonMixin):
+    """
+    Simple container for asset filtering that focuses on in-memory filtering for REST APIs.
+
+    This class provides basic asset filtering functionality without SQL generation complexity.
+    It's designed for the initial phase where assets are collected via REST APIs rather than
+    SQL queries.
+
+    Example usage:
+        # Block all external assets
+        filters = AssetAllowBlockList(
+            filters=[AssetFilter(asset_type="external", effect=FilterEffectType.BLOCK)]
+        )
+        container = AssetFiltersContainer(asset_filters=filters)
+
+        # Check if an asset is blocked
+        is_blocked = container.is_asset_blocked("external", "my_table")  # True
+        is_blocked = container.is_asset_blocked("table", "users")        # False
+    """
+
+    asset_filters: List[AssetAllowBlockList] = field(default_factory=list)
+
+    def is_asset_type_filtered(self, asset_type: str) -> bool:
+        """Returns True if any filters are configured for the given asset type."""
+        return bool(self._get_asset_filters(asset_type))
+
+    def is_asset_blocked(self, asset_type: str, attributes: Dict[str, str]) -> bool:
+        """
+        Returns True if the specified asset is blocked by the current filters.
+
+        Args:
+          asset_type: The type of asset (e.g., 'tableau_workbook_v2', 'jobs', 'power_bi_workspace')
+          attributes: A dictionary representing the attributes of the asset
+
+        Returns:
+          True if the asset is blocked, False if it's allowed
+        """
+        asset_filters = self._get_asset_filters(asset_type)
+
+        is_blocked = False
+
+        for asset_filter in asset_filters:
+            default_effect_matches = asset_filter.get_default_effect_rules(
+                lambda f: f.matches(force_regexp=False, **attributes)
+            )
+            if default_effect_matches:
+                is_blocked = asset_filter.default_effect == FilterEffectType.BLOCK
+            else:
+                other_effect_matches = asset_filter.get_other_effect_rules(
+                    lambda f: f.matches(force_regexp=False, **attributes)
+                )
+                if other_effect_matches:
+                    is_blocked = asset_filter.other_effect == FilterEffectType.BLOCK
+                else:
+                    # No matches, use default effect
+                    is_blocked = asset_filter.default_effect == FilterEffectType.BLOCK
+
+        return is_blocked
+
+    def _get_asset_filters(self, asset_type: str) -> List[AssetAllowBlockList]:
+        return [f for f in self.asset_filters if f.asset_type == asset_type]

pycarlo/features/metadata/base_allow_block_list.py

@@ -0,0 +1,137 @@
+import enum
+import re
+from dataclasses import dataclass, field
+from typing import Any, Callable, Generic, List, Optional, TypeVar
+
+from dataclasses_json import DataClassJsonMixin
+
+from pycarlo.common import get_logger
+
+logger = get_logger(__name__)
+
+# For documentation and samples check the link below:
+# https://www.notion.so/montecarlodata/Catalog-Schema-Filtering-59edd6eff7f74c94ab6bfca75d2e3ff1
+
+
+def _exclude_none_values(value: Any) -> bool:
+    return value is None
+
+
+class FilterEffectType(enum.Enum):
+    BLOCK = "block"
+    ALLOW = "allow"
+
+
+RuleEffect = FilterEffectType
+
+
+class FilterType(enum.Enum):
+    EXACT_MATCH = "exact_match"
+    PREFIX = "prefix"
+    SUFFIX = "suffix"
+    SUBSTRING = "substring"
+    REGEXP = "regexp"
+
+
+ComparisonType = FilterType
+
+# Type variable for the filter class
+FilterRuleT = TypeVar("FilterRuleT", bound="FilterRule")
+
+
+@dataclass
+class RuleCondition(DataClassJsonMixin):
+    attribute_name: str
+    value: str
+    comparison_type: ComparisonType = ComparisonType.EXACT_MATCH
+
+
+@dataclass
+class FilterRule(DataClassJsonMixin):
+    """
+    Base class for all filter types. Provides common filtering logic that can be
+    shared between different filter implementations (e.g., metadata filters, asset filters).
+    """
+
+    conditions: Optional[List[RuleCondition]] = field(default_factory=list)
+    effect: RuleEffect = RuleEffect.BLOCK
+
+    def matches(self, force_regexp: bool = False, **kwargs: Any) -> bool:
+        """
+        Returns True if all properties specified in kwargs match the conditions specified in
+        properties of the same name in this object.
+        If any of the conditions (for example self.field) is None, that condition will be matched.
+        """
+        if not kwargs:
+            raise ValueError("At least one field needs to be specified for matching")
+
+        # kwargs must match the field names in this class, if any of them do not,
+        # invalidate the filter.
+        try:
+            return all(
+                condition.attribute_name not in kwargs
+                or self._match(
+                    condition=condition,
+                    value=kwargs.get(condition.attribute_name),
+                    force_regexp=force_regexp,
+                )
+                for condition in self.conditions or []
+            )
+        except AttributeError:
+            return False
+
+    @classmethod
+    def _match(cls, condition: RuleCondition, value: Optional[str], force_regexp: bool) -> bool:
+        # Field not specified on this object, e.g. self.field=None, which matches everything
+        if value is None:
+            return False
+
+        # The comparison is performed case-insensitive (check BaseFilter._safe_match)
+        # We can use LOWER here since it is part of standard SQL (like AND/OR/NOT), so including it
+        # here is a way to make sure that all comparisons are case-insensitive in the SQL sentences
+        # for all engines. Added option to not always LOWER since customers do have lower/upper case
+        # databases logged in MC
+        filter_value = condition.value.lower()
+        value = value.lower()
+
+        if force_regexp or condition.comparison_type == FilterType.REGEXP:
+            regexp = f"^{filter_value}$"
+            return re.match(regexp, value) is not None
+        elif condition.comparison_type == FilterType.PREFIX:
+            return value.startswith(filter_value)
+        elif condition.comparison_type == FilterType.SUFFIX:
+            return value.endswith(filter_value)
+        elif condition.comparison_type == FilterType.SUBSTRING:
+            return filter_value in value
+        else:  # filter_type == FilterType.EXACT_MATCH
+            return filter_value == value
+
+
+@dataclass
+class BaseAllowBlockList(Generic[FilterRuleT], DataClassJsonMixin):
+    rules: Optional[List[FilterRuleT]] = field(default_factory=list)
+    default_effect: RuleEffect = RuleEffect.ALLOW
+
+    @property
+    def other_effect(self) -> RuleEffect:
+        return RuleEffect.ALLOW if self.default_effect == RuleEffect.BLOCK else RuleEffect.BLOCK
+
+    def get_default_effect_rules(
+        self, condition: Optional[Callable[[FilterRuleT], bool]] = None
+    ) -> List[FilterRuleT]:
+        return list(
+            filter(
+                lambda f: f.effect == self.default_effect and (condition is None or condition(f)),
+                self.rules or [],
+            )
+        )
+
+    def get_other_effect_rules(
+        self, condition: Optional[Callable[[FilterRuleT], bool]] = None
+    ) -> List[FilterRuleT]:
+        return list(
+            filter(
+                lambda f: f.effect != self.default_effect and (condition is None or condition(f)),
+                self.rules or [],
+            )
+        )

pycarlo/features/metadata/metadata_allow_block_list.py

@@ -0,0 +1,94 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from dataclasses_json import config, dataclass_json
+
+from pycarlo.common import get_logger
+from pycarlo.features.metadata.base_allow_block_list import (
+    BaseAllowBlockList,
+    ComparisonType,
+    FilterRule,
+    FilterType,
+    RuleCondition,
+)
+
+logger = get_logger(__name__)
+
+# For documentation and samples check the link below:
+# https://www.notion.so/montecarlodata/Catalog-Schema-Filtering-59edd6eff7f74c94ab6bfca75d2e3ff1
+
+
+@dataclass_json
+@dataclass
+class MetadataFilter(FilterRule):
+    type: FilterType = FilterType.EXACT_MATCH
+
+    # we're using exclude=_exclude_none_values to prevent these properties to be serialized to json
+    # when None, to keep the json doc simpler
+    project: Optional[str] = field(metadata=config(exclude=lambda x: x is None), default=None)
+    dataset: Optional[str] = field(metadata=config(exclude=lambda x: x is None), default=None)
+    table_type: Optional[str] = field(metadata=config(exclude=lambda x: x is None), default=None)
+    table_name: Optional[str] = field(metadata=config(exclude=lambda x: x is None), default=None)
+
+    def __post_init__(self):
+        # For backwards compatibility, we now create a set of conditions based on the
+        # metadata-specific fields.
+        self.conditions = self.conditions or []
+        if self.table_name is not None:
+            is_target_field = self.filter_type_target_field() == "table_name"
+            condition = RuleCondition(
+                comparison_type=self.type if is_target_field else ComparisonType.EXACT_MATCH,
+                attribute_name="table_name",
+                value=self.table_name,
+            )
+            self.conditions.append(condition)
+
+        if self.dataset is not None:
+            is_target_field = self.filter_type_target_field() == "dataset"
+            condition = RuleCondition(
+                comparison_type=self.type if is_target_field else ComparisonType.EXACT_MATCH,
+                attribute_name="dataset",
+                value=self.dataset,
+            )
+            self.conditions.append(condition)
+
+        if self.project is not None:
+            is_target_field = self.filter_type_target_field() == "project"
+            condition = RuleCondition(
+                comparison_type=self.type if is_target_field else ComparisonType.EXACT_MATCH,
+                attribute_name="project",
+                value=self.project,
+            )
+            self.conditions.append(condition)
+
+        if self.table_type is not None:
+            condition = RuleCondition(
+                comparison_type=ComparisonType.EXACT_MATCH,
+                attribute_name="table_type",
+                value=self.table_type,
+            )
+            self.conditions.append(condition)
+
+    def filter_type_target_field(self) -> str:
+        """
+        The field that is evaluated using filter type. Other fields should be
+        compared using exact match.
+        """
+        if self.table_name is not None:
+            return "table_name"
+        if self.dataset is not None:
+            return "dataset"
+        if self.project is not None:
+            return "project"
+
+        logger.exception("Invalid filter, missing target values")
+        return ""
+
+
+@dataclass_json
+@dataclass
+class MetadataAllowBlockList(BaseAllowBlockList[MetadataFilter]):
+    filters: List[MetadataFilter] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.rules = self.filters

pycarlo/features/metadata/metadata_filters_container.py

@@ -3,7 +3,12 @@ from typing import Any, Callable, Dict, Optional
 
 from dataclasses_json import dataclass_json
 
-from pycarlo.features.metadata import AllowBlockList, FilterEffectType, FilterType, MetadataFilter
+from pycarlo.features.metadata import (
+    FilterEffectType,
+    FilterType,
+    MetadataAllowBlockList,
+    MetadataFilter,
+)
 
 
 @dataclass_json
@@ -62,7 +67,7 @@ class MetadataFiltersContainer:
         - allowed: (project_3, dataset_1), (project_4, dataset_4)
     """
 
-    metadata_filters: AllowBlockList = field(default_factory=AllowBlockList)
+    metadata_filters: MetadataAllowBlockList = field(default_factory=MetadataAllowBlockList)
 
     @property
     def is_metadata_filtered(self) -> bool:
@@ -129,7 +134,7 @@ class MetadataFiltersContainer:
 
     @staticmethod
     def _get_effect(
-        metadata_filters: AllowBlockList,
+        metadata_filters: MetadataAllowBlockList,
         force_regexp: bool,
         condition: Optional[Callable[[MetadataFilter], bool]] = None,
         **kwargs: Any,
@@ -145,13 +150,13 @@ class MetadataFiltersContainer:
         """
         if not metadata_filters.filters or any(
             f.matches(force_regexp, **kwargs)
-            for f in metadata_filters.get_default_effect_filters(condition=condition)
+            for f in metadata_filters.get_default_effect_rules(condition=condition)
         ):
             return metadata_filters.default_effect
 
         if any(
             f.matches(force_regexp, **kwargs)
-            for f in metadata_filters.get_other_effect_filters(condition=condition)
+            for f in metadata_filters.get_other_effect_rules(condition=condition)
         ):
             return metadata_filters.other_effect
 
@@ -198,10 +203,10 @@ class MetadataFiltersContainer:
             return not project or f.matches(project=project)
 
         default_effect = self.metadata_filters.default_effect
-        default_effect_filters = self.metadata_filters.get_default_effect_filters(
+        default_effect_filters = self.metadata_filters.get_default_effect_rules(
             condition=project_condition
         )
-        other_effect_filters = self.metadata_filters.get_other_effect_filters(
+        other_effect_filters = self.metadata_filters.get_other_effect_rules(
             condition=project_condition
         )
         default_effect_op = " OR " if default_effect == FilterEffectType.ALLOW else " AND "
@@ -209,7 +214,7 @@ class MetadataFiltersContainer:
 
         default_effect_conditions = default_effect_op.join(
             [
-                f"({self._get_sql_field_condition(f, column_mapping, encoder, force_lowercase)})"
+                self._get_sql_field_condition(f, column_mapping, encoder, force_lowercase)
                 for f in default_effect_filters
             ]
         )
@@ -219,13 +224,15 @@ class MetadataFiltersContainer:
                 for f in other_effect_filters
             ]
         )
-        conditions = default_effect_conditions
-        if conditions and other_effect_conditions:
-            conditions += default_effect_op
-            conditions += "(" + other_effect_conditions + ")"
-        elif not conditions:
-            conditions = other_effect_conditions
-        return f"({conditions})" if conditions else ""
+
+        if default_effect_conditions and other_effect_conditions:
+            return f"(({default_effect_conditions}){default_effect_op}({other_effect_conditions}))"
+        elif default_effect_conditions:
+            return f"({default_effect_conditions})"
+        elif other_effect_conditions:
+            return f"({other_effect_conditions})"
+        else:
+            return None
 
     @staticmethod
     def _get_sql_field_condition(
@@ -233,7 +240,7 @@ class MetadataFiltersContainer:
         column_mapping: Dict,
         encoder: Callable[[str, str, FilterType], str],
         force_lowercase: Optional[bool] = True,
-    ):
+    ) -> str:
         # The comparison is performed case-insensitive (check MetadataFilter._safe_match)
         # We can use LOWER here since it is part of standard SQL (like AND/OR/NOT), so including it
         # here is a way to make sure that all comparisons are case-insensitive in the SQL sentences
@@ -250,4 +257,6 @@ class MetadataFiltersContainer:
                 if getattr(mf, field) is not None
             ]
         )
+        if not conditions:
+            return ""
         return f"NOT({conditions})" if mf.effect == FilterEffectType.BLOCK else f"({conditions})"

pycarlo/lib/README.md

@@ -1,3 +1,35 @@
-These files are auto-generated. **Do not edit**!
+# Monte Carlo GraphQL Schema Library
 
-Use `make generate` to update the schema (e.g. support new/modified queries & mutations).
+The `schema.json` and `schema.py` files are auto-generated. **Do not edit them directly**!
+
+If you need to customize the schema, see below. Refer to the
+[CONTRIBUTING.md](../../CONTRIBUTING.md) for general development guidelines.
+
+## Schema Customizations
+
+The generated `schema.py` is automatically modified during the build process to apply the following
+customizations. This is done via `sed` commands in the [Makefile](../../Makefile), but if we need to
+get fancier, we just can update the `customize-schema` target there to call whatever we need to do.
+
+### Connection Type Fix
+
+The `Connection` class is changed from `sgqlc.types.relay.Connection` to `sgqlc.types.Type`.
+
+**Why:** sgqlc automatically makes all types ending in "Connection" inherit from `relay.Connection`,
+which makes `Connection` not a valid field type. This causes requests to fail when attempting to
+resolve it. Changing it to inherit from `sgqlc.types.Type` fixes this issue.
+
+[Related PR](https://github.com/monte-carlo-data/python-sdk/pull/63)
+
+### Backward-Compatible Enums
+
+All GraphQL enum types use `pycarlo.lib.types.Enum` instead of `sgqlc.types.Enum`. This custom enum
+class gracefully handles unknown enum values by returning them as strings instead of raising errors.
+
+**Why:** When new enum values are added to the Monte Carlo API, older SDK versions would crash when
+deserializing responses containing these new values. Our custom Enum prevents this by:
+
+- Returning unknown values as plain strings (same type as known values)
+- Logging a warning when unknown values are encountered
+
+See [pycarlo/lib/types.py](types.py) for implementation details.