acryl-datahub 1.1.0.4rc1__py3-none-any.whl → 1.1.0.4rc3__py3-none-any.whl

datahub/ingestion/source/ge_data_profiler.py

@@ -120,7 +120,6 @@ SNOWFLAKE = "snowflake"
 BIGQUERY = "bigquery"
 REDSHIFT = "redshift"
 DATABRICKS = "databricks"
-TRINO = "trino"
 
 # Type names for Databricks, to match Title Case types in sqlalchemy
 ProfilerTypeMapping.INT_TYPE_NAMES.append("Integer")
@@ -206,6 +205,17 @@ def get_column_unique_count_dh_patch(self: SqlAlchemyDataset, column: str) -> in
             )
         )
         return convert_to_json_serializable(element_values.fetchone()[0])
+    elif (
+        self.engine.dialect.name.lower() == GXSqlDialect.AWSATHENA
+        or self.engine.dialect.name.lower() == GXSqlDialect.TRINO
+    ):
+        return convert_to_json_serializable(
+            self.engine.execute(
+                sa.select(sa.func.approx_distinct(sa.column(column))).select_from(
+                    self._table
+                )
+            ).scalar()
+        )
     return convert_to_json_serializable(
         self.engine.execute(
             sa.select([sa.func.count(sa.func.distinct(sa.column(column)))]).select_from(
@@ -734,11 +744,41 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase):
     def _get_dataset_column_distinct_value_frequencies(
         self, column_profile: DatasetFieldProfileClass, column: str
     ) -> None:
-        if self.config.include_field_distinct_value_frequencies:
+        if not self.config.include_field_distinct_value_frequencies:
+            return
+        try:
+            results = self.dataset.engine.execute(
+                sa.select(
+                    [
+                        sa.column(column),
+                        sa.func.count(sa.column(column)),
+                    ]
+                )
+                .select_from(self.dataset._table)
+                .where(sa.column(column).is_not(None))
+                .group_by(sa.column(column))
+            ).fetchall()
+
             column_profile.distinctValueFrequencies = [
-                ValueFrequencyClass(value=str(value), frequency=count)
-                for value, count in self.dataset.get_column_value_counts(column).items()
+                ValueFrequencyClass(value=str(value), frequency=int(count))
+                for value, count in results
             ]
+            # sort so output is deterministic. don't do it in SQL because not all column
+            # types are sortable in SQL (such as JSON data types on Athena/Trino).
+            column_profile.distinctValueFrequencies = sorted(
+                column_profile.distinctValueFrequencies, key=lambda x: x.value
+            )
+        except Exception as e:
+            logger.debug(
+                f"Caught exception while attempting to get distinct value frequencies for column {column}. {e}"
+            )
+
+            self.report.report_warning(
+                title="Profiling: Unable to Calculate Distinct Value Frequencies",
+                message="Distinct value frequencies for the column will not be accessible",
+                context=f"{self.dataset_name}.{column}",
+                exc=e,
+            )
 
     @_run_with_query_combiner
     def _get_dataset_column_histogram(
@@ -1395,12 +1435,12 @@ class DatahubGEProfiler:
                     )
                 return None
             finally:
-                if batch is not None and self.base_engine.engine.name.upper() in [
-                    "TRINO",
-                    "AWSATHENA",
+                if batch is not None and self.base_engine.engine.name.lower() in [
+                    GXSqlDialect.TRINO,
+                    GXSqlDialect.AWSATHENA,
                 ]:
                     if (
-                        self.base_engine.engine.name.upper() == "TRINO"
+                        self.base_engine.engine.name.lower() == GXSqlDialect.TRINO
                         or temp_view is not None
                     ):
                         self._drop_temp_table(batch)

datahub/ingestion/source/iceberg/iceberg.py

@@ -134,7 +134,9 @@ logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
     SourceCapability.OWNERSHIP,
     "Automatically ingests ownership information from table properties based on `user_ownership_property` and `group_ownership_property`",
 )
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 class IcebergSource(StatefulIngestionSourceBase):
     """
     ## Integration Details

datahub/ingestion/source/kafka/kafka.py

@@ -204,6 +204,7 @@ class KafkaConnectionTest:
     "Not supported",
     supported=False,
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class KafkaSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin extracts the following:

datahub/ingestion/source/looker/looker_source.py

@@ -126,6 +126,7 @@ logger = logging.getLogger(__name__)
     SourceCapability.USAGE_STATS,
     "Enabled by default, configured using `extract_usage_history`",
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
     """
     This plugin extracts the following:

datahub/ingestion/source/powerbi/powerbi.py

@@ -1253,6 +1253,7 @@ class Mapper:
     SourceCapability.DATA_PROFILING,
     "Optionally enabled via configuration profiling.enabled",
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class PowerBiDashboardSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin extracts the following:

datahub/ingestion/source/qlik_sense/qlik_sense.py

@@ -109,6 +109,7 @@ logger = logging.getLogger(__name__)
     "Enabled by default, configured using `ingest_owner`",
 )
 @capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class QlikSenseSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin extracts the following:

datahub/ingestion/source/redshift/redshift.py

@@ -10,6 +10,7 @@ import humanfriendly
 import pydantic
 import redshift_connector
 
+from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.pattern_utils import is_schema_allowed
 from datahub.emitter.mce_builder import (
     make_data_platform_urn,
@@ -140,12 +141,15 @@ logger: logging.Logger = logging.getLogger(__name__)
     SourceCapability.USAGE_STATS,
     "Enabled by default, can be disabled via configuration `include_usage_statistics`",
 )
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 @capability(
     SourceCapability.CLASSIFICATION,
     "Optionally enabled via `classification.enabled`",
     supported=True,
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class RedshiftSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin extracts the following:
@@ -354,7 +358,23 @@ class RedshiftSource(StatefulIngestionSourceBase, TestableSource):
             ).workunit_processor,
         ]
 
+    def _warn_deprecated_configs(self):
+        if (
+            self.config.match_fully_qualified_names is not None
+            and not self.config.match_fully_qualified_names
+            and self.config.schema_pattern is not None
+            and self.config.schema_pattern != AllowDenyPattern.allow_all()
+        ):
+            self.report.report_warning(
+                message="Please update `schema_pattern` to match against fully qualified schema name `<database_name>.<schema_name>` and set config `match_fully_qualified_names : True`."
+                "Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. "
+                "The config option `match_fully_qualified_names` will be removed in future and the default behavior will be like `match_fully_qualified_names: True`.",
+                context="Config option deprecation warning",
+                title="Config option deprecation warning",
+            )
+
     def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]:
+        self._warn_deprecated_configs()
         connection = self._try_get_redshift_connection(self.config)
 
         if connection is None:

datahub/ingestion/source/sac/sac.py

@@ -178,7 +178,9 @@ class SACSourceReport(StaleEntityRemovalSourceReport):
     SourceCapability.LINEAGE_COARSE,
     "Enabled by default (only for Live Data Models)",
 )
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 @capability(
     SourceCapability.SCHEMA_METADATA,
     "Enabled by default (only for Import Data Models)",

datahub/ingestion/source/sigma/sigma.py

@@ -105,6 +105,7 @@ logger = logging.getLogger(__name__)
     SourceCapability.OWNERSHIP,
     "Enabled by default, configured using `ingest_owner`",
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class SigmaSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin extracts the following:

datahub/ingestion/source/snowflake/snowflake_config.py

@@ -154,14 +154,11 @@ class SnowflakeIdentifierConfig(
 
     email_domain: Optional[str] = pydantic.Field(
         default=None,
-        description="Email domain of your organization so users can be displayed on UI appropriately.",
+        description="Email domain of your organization so users can be displayed on UI appropriately. This is used only if we cannot infer email ID.",
     )
 
-    email_as_user_identifier: bool = Field(
-        default=True,
-        description="Format user urns as an email, if the snowflake user's email is set. If `email_domain` is "
-        "provided, generates email addresses for snowflake users with unset emails, based on their "
-        "username.",
+    _email_as_user_identifier = pydantic_removed_field(
+        "email_as_user_identifier",
     )
 
 

datahub/ingestion/source/snowflake/snowflake_utils.py

@@ -325,15 +325,10 @@ class SnowflakeIdentifierBuilder:
         user_email: Optional[str],
     ) -> str:
         if user_email:
-            return self.snowflake_identifier(
-                user_email
-                if self.identifier_config.email_as_user_identifier is True
-                else user_email.split("@")[0]
-            )
+            return self.snowflake_identifier(user_email)
         return self.snowflake_identifier(
             f"{user_name}@{self.identifier_config.email_domain}"
-            if self.identifier_config.email_as_user_identifier is True
-            and self.identifier_config.email_domain is not None
+            if self.identifier_config.email_domain is not None
             else user_name
         )
 

datahub/ingestion/source/snowflake/snowflake_v2.py

@@ -131,6 +131,7 @@ logger: logging.Logger = logging.getLogger(__name__)
     "Optionally enabled via `classification.enabled`",
     supported=True,
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class SnowflakeV2Source(
     SnowflakeCommonMixin,
     StatefulIngestionSourceBase,
@@ -311,6 +312,7 @@ class SnowflakeV2Source(
                 SourceCapability.PLATFORM_INSTANCE,
                 SourceCapability.DOMAINS,
                 SourceCapability.DELETION_DETECTION,
+                SourceCapability.TEST_CONNECTION,
             )
         ]
 

datahub/ingestion/source/sql/clickhouse.py

@@ -379,7 +379,9 @@ clickhouse_datetime_format = "%Y-%m-%d %H:%M:%S"
 @platform_name("ClickHouse")
 @config_class(ClickHouseConfig)
 @support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
 class ClickHouseSource(TwoTierSQLAlchemySource):
     """

datahub/ingestion/source/sql/cockroachdb.py

@@ -26,7 +26,6 @@ class CockroachDBConfig(PostgresConfig):
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
 class CockroachDBSource(PostgresSource):
     config: CockroachDBConfig
 

datahub/ingestion/source/sql/hana.py

@@ -27,7 +27,9 @@ class HanaConfig(BasicSQLAlchemyConfig):
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 class HanaSource(SQLAlchemySource):
     def __init__(self, config: HanaConfig, ctx: PipelineContext):
         super().__init__(config, ctx, "hana")

datahub/ingestion/source/sql/hive_metastore.py

@@ -161,7 +161,9 @@ class HiveMetastore(BasicSQLAlchemyConfig):
 @platform_name("Hive Metastore")
 @config_class(HiveMetastore)
 @support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 @capability(SourceCapability.DATA_PROFILING, "Not Supported", False)
 @capability(SourceCapability.CLASSIFICATION, "Not Supported", False)
 @capability(

datahub/ingestion/source/sql/mariadb.py

@@ -15,7 +15,6 @@ from datahub.ingestion.source.sql.mysql import MySQLConfig, MySQLSource
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
 class MariaDBSource(MySQLSource):
     def get_platform(self):
         return "mariadb"

datahub/ingestion/source/sql/mssql/source.py

@@ -174,7 +174,14 @@ class SQLServerConfig(BasicSQLAlchemyConfig):
 @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.LINEAGE_COARSE,
+    "Enabled by default to get lineage for stored procedures via `include_lineage` and for views via `include_view_lineage`",
+)
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Enabled by default to get lineage for stored procedures via `include_lineage` and for views via `include_view_column_lineage`",
+)
 class SQLServerSource(SQLAlchemySource):
     """
     This plugin extracts the following:

datahub/ingestion/source/sql/mysql.py

@@ -65,15 +65,6 @@ class MySQLConfig(MySQLConnectionConfig, TwoTierSQLAlchemyConfig):
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
-@capability(
-    SourceCapability.LINEAGE_COARSE,
-    "Supported for views if `include_view_column_lineage` is enabled.",
-)
-@capability(
-    SourceCapability.LINEAGE_FINE,
-    "Supported for views if `include_view_column_lineage` is enabled.",
-)
 class MySQLSource(TwoTierSQLAlchemySource):
     """
     This plugin extracts the following:

datahub/ingestion/source/sql/postgres.py

@@ -131,7 +131,6 @@ class PostgresConfig(BasePostgresConfig):
 @capability(SourceCapability.DOMAINS, "Enabled by default")
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
-@capability(SourceCapability.LINEAGE_COARSE, "Optionally enabled via configuration")
 class PostgresSource(SQLAlchemySource):
     """
     This plugin extracts the following:

datahub/ingestion/source/sql/sql_common.py

@@ -302,6 +302,18 @@ class ProfileMetadata:
     "Enabled by default",
     supported=True,
 )
+@capability(
+    SourceCapability.LINEAGE_COARSE,
+    "Enabled by default to get lineage for views via `include_view_lineage`",
+)
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Enabled by default to get lineage for views via `include_view_column_lineage`",
+)
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 class SQLAlchemySource(StatefulIngestionSourceBase, TestableSource):
     """A Base class for all SQL Sources that use SQLAlchemy to extend"""
 

datahub/ingestion/source/tableau/tableau.py

@@ -879,6 +879,7 @@ def report_user_role(report: TableauSourceReport, server: Server) -> None:
     SourceCapability.LINEAGE_FINE,
     "Enabled by default, configure using `extract_column_level_lineage`",
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class TableauSource(StatefulIngestionSourceBase, TestableSource):
     platform = "tableau"
 

datahub/ingestion/source/unity/source.py

@@ -162,6 +162,7 @@ logger: logging.Logger = logging.getLogger(__name__)
     "Optionally enabled via `stateful_ingestion.remove_stale_metadata`",
     supported=True,
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 @support_status(SupportStatus.INCUBATING)
 class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
     """

datahub/ingestion/source/usage/clickhouse_usage.py

@@ -85,8 +85,11 @@ class ClickHouseUsageConfig(ClickHouseConfig, BaseUsageConfig, EnvConfigMixin):
 @platform_name("ClickHouse")
 @config_class(ClickHouseUsageConfig)
 @support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
+@capability(SourceCapability.USAGE_STATS, "Enabled by default to get usage stats")
 @dataclasses.dataclass
 class ClickHouseUsageSource(Source):
     """

datahub/ingestion/source/usage/starburst_trino_usage.py

@@ -15,7 +15,9 @@ from sqlalchemy.engine import Engine
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.time_window_config import get_time_bucket
 from datahub.ingestion.api.decorators import (
+    SourceCapability,
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
@@ -112,6 +114,7 @@ class TrinoUsageReport(SourceReport):
 @platform_name("Trino")
 @config_class(TrinoUsageConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.USAGE_STATS, "Enabled by default to get usage stats")
 @dataclasses.dataclass
 class TrinoUsageSource(Source):
     """

datahub/metadata/_internal_schema_classes.py

@@ -4590,6 +4590,15 @@ class FabricTypeClass(object):
     RVW = "RVW"
     """Designates review fabrics"""
     
+    PRD = "PRD"
+    """Alternative Prod spelling"""
+    
+    TST = "TST"
+    """Alternative Test spelling"""
+    
+    SIT = "SIT"
+    """System Integration Testing"""
+    
     SANDBOX = "SANDBOX"
     """Designates sandbox fabrics"""
     
@@ -21504,6 +21513,7 @@ class DataHubResourceFilterClass(DictWrapper):
         resources: Union[None, List[str]]=None,
         allResources: Optional[bool]=None,
         filter: Union[None, "PolicyMatchFilterClass"]=None,
+        privilegeConstraints: Union[None, "PolicyMatchFilterClass"]=None,
     ):
         super().__init__()
         
@@ -21515,12 +21525,14 @@ class DataHubResourceFilterClass(DictWrapper):
         else:
             self.allResources = allResources
         self.filter = filter
+        self.privilegeConstraints = privilegeConstraints
     
     def _restore_defaults(self) -> None:
         self.type = self.RECORD_SCHEMA.fields_dict["type"].default
         self.resources = self.RECORD_SCHEMA.fields_dict["resources"].default
         self.allResources = self.RECORD_SCHEMA.fields_dict["allResources"].default
         self.filter = self.RECORD_SCHEMA.fields_dict["filter"].default
+        self.privilegeConstraints = self.RECORD_SCHEMA.fields_dict["privilegeConstraints"].default
     
     
     @property
@@ -21565,6 +21577,16 @@ class DataHubResourceFilterClass(DictWrapper):
         self._inner_dict['filter'] = value
     
     
+    @property
+    def privilegeConstraints(self) -> Union[None, "PolicyMatchFilterClass"]:
+        """Constraints around what sub-resources operations are allowed to modify, i.e. NOT_EQUALS - cannot modify a particular defined tag, EQUALS - can only modify a particular defined tag, STARTS_WITH - can only modify a tag starting with xyz"""
+        return self._inner_dict.get('privilegeConstraints')  # type: ignore
+    
+    @privilegeConstraints.setter
+    def privilegeConstraints(self, value: Union[None, "PolicyMatchFilterClass"]) -> None:
+        self._inner_dict['privilegeConstraints'] = value
+    
+    
 class DataHubRoleInfoClass(_Aspect):
     """Information about a DataHub Role."""
 
@@ -21633,6 +21655,9 @@ class PolicyMatchConditionClass(object):
     STARTS_WITH = "STARTS_WITH"
     """Whether the field value starts with the value"""
     
+    NOT_EQUALS = "NOT_EQUALS"
+    """Whether the field does not match the value"""
+    
     
     
 class PolicyMatchCriterionClass(DictWrapper):

datahub/metadata/schema.avsc

@@ -9502,13 +9502,16 @@
               "DEV": "Designates development fabrics",
               "EI": "Designates early-integration fabrics",
               "NON_PROD": "Designates non-production fabrics",
+              "PRD": "Alternative Prod spelling",
               "PRE": "Designates pre-production fabrics",
               "PROD": "Designates production fabrics",
               "QA": "Designates quality assurance fabrics",
               "RVW": "Designates review fabrics",
               "SANDBOX": "Designates sandbox fabrics",
+              "SIT": "System Integration Testing",
               "STG": "Designates staging fabrics",
               "TEST": "Designates testing fabrics",
+              "TST": "Alternative Test spelling",
               "UAT": "Designates user acceptance testing fabrics"
             },
             "name": "FabricType",
@@ -9525,6 +9528,9 @@
               "PROD",
               "CORP",
               "RVW",
+              "PRD",
+              "TST",
+              "SIT",
               "SANDBOX"
             ],
             "doc": "Fabric group type"
@@ -16441,13 +16447,15 @@
                                                     "type": "enum",
                                                     "symbolDocs": {
                                                       "EQUALS": "Whether the field matches the value",
+                                                      "NOT_EQUALS": "Whether the field does not match the value",
                                                       "STARTS_WITH": "Whether the field value starts with the value"
                                                     },
                                                     "name": "PolicyMatchCondition",
                                                     "namespace": "com.linkedin.pegasus2avro.policy",
                                                     "symbols": [
                                                       "EQUALS",
-                                                      "STARTS_WITH"
+                                                      "STARTS_WITH",
+                                                      "NOT_EQUALS"
                                                     ],
                                                     "doc": "The matching condition in a filter criterion"
                                                   },
@@ -16469,6 +16477,15 @@
                                   "name": "filter",
                                   "default": null,
                                   "doc": "Filter to apply privileges to"
+                                },
+                                {
+                                  "type": [
+                                    "null",
+                                    "com.linkedin.pegasus2avro.policy.PolicyMatchFilter"
+                                  ],
+                                  "name": "privilegeConstraints",
+                                  "default": null,
+                                  "doc": "Constraints around what sub-resources operations are allowed to modify, i.e. NOT_EQUALS - cannot modify a particular defined tag, EQUALS - can only modify a particular defined tag, STARTS_WITH - can only modify a tag starting with xyz"
                                 }
                               ],
                               "doc": "Information used to filter DataHub resource."

datahub/metadata/schemas/ContainerProperties.avsc

@@ -93,13 +93,16 @@
             "DEV": "Designates development fabrics",
             "EI": "Designates early-integration fabrics",
             "NON_PROD": "Designates non-production fabrics",
+            "PRD": "Alternative Prod spelling",
             "PRE": "Designates pre-production fabrics",
             "PROD": "Designates production fabrics",
             "QA": "Designates quality assurance fabrics",
             "RVW": "Designates review fabrics",
             "SANDBOX": "Designates sandbox fabrics",
+            "SIT": "System Integration Testing",
             "STG": "Designates staging fabrics",
             "TEST": "Designates testing fabrics",
+            "TST": "Alternative Test spelling",
             "UAT": "Designates user acceptance testing fabrics"
           },
           "name": "FabricType",
@@ -116,6 +119,9 @@
             "PROD",
             "CORP",
             "RVW",
+            "PRD",
+            "TST",
+            "SIT",
             "SANDBOX"
           ],
           "doc": "Fabric group type"

datahub/metadata/schemas/DataFlowInfo.avsc

@@ -147,13 +147,16 @@
             "DEV": "Designates development fabrics",
             "EI": "Designates early-integration fabrics",
             "NON_PROD": "Designates non-production fabrics",
+            "PRD": "Alternative Prod spelling",
             "PRE": "Designates pre-production fabrics",
             "PROD": "Designates production fabrics",
             "QA": "Designates quality assurance fabrics",
             "RVW": "Designates review fabrics",
             "SANDBOX": "Designates sandbox fabrics",
+            "SIT": "System Integration Testing",
             "STG": "Designates staging fabrics",
             "TEST": "Designates testing fabrics",
+            "TST": "Alternative Test spelling",
             "UAT": "Designates user acceptance testing fabrics"
           },
           "name": "FabricType",
@@ -170,6 +173,9 @@
             "PROD",
             "CORP",
             "RVW",
+            "PRD",
+            "TST",
+            "SIT",
             "SANDBOX"
           ],
           "doc": "Fabric group type"

datahub/metadata/schemas/DataHubPolicyInfo.avsc

@@ -110,13 +110,15 @@
                                 "type": "enum",
                                 "symbolDocs": {
                                   "EQUALS": "Whether the field matches the value",
+                                  "NOT_EQUALS": "Whether the field does not match the value",
                                   "STARTS_WITH": "Whether the field value starts with the value"
                                 },
                                 "name": "PolicyMatchCondition",
                                 "namespace": "com.linkedin.pegasus2avro.policy",
                                 "symbols": [
                                   "EQUALS",
-                                  "STARTS_WITH"
+                                  "STARTS_WITH",
+                                  "NOT_EQUALS"
                                 ],
                                 "doc": "The matching condition in a filter criterion"
                               },
@@ -138,6 +140,15 @@
               "name": "filter",
               "default": null,
               "doc": "Filter to apply privileges to"
+            },
+            {
+              "type": [
+                "null",
+                "com.linkedin.pegasus2avro.policy.PolicyMatchFilter"
+              ],
+              "name": "privilegeConstraints",
+              "default": null,
+              "doc": "Constraints around what sub-resources operations are allowed to modify, i.e. NOT_EQUALS - cannot modify a particular defined tag, EQUALS - can only modify a particular defined tag, STARTS_WITH - can only modify a tag starting with xyz"
             }
           ],
           "doc": "Information used to filter DataHub resource."