acryl-datahub 1.1.0rc3__py3-none-any.whl → 1.1.0.1rc6__py3-none-any.whl

datahub/sdk/main_client.py

@@ -7,15 +7,16 @@ from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
 from datahub.ingestion.graph.config import ClientMode, DatahubClientConfig
 from datahub.sdk.entity_client import EntityClient
 from datahub.sdk.lineage_client import LineageClient
-from datahub.sdk.resolver_client import ResolverClient
 from datahub.sdk.search_client import SearchClient
 
 try:
     from acryl_datahub_cloud._sdk_extras import (  # type: ignore[import-not-found]
-        AssertionsClient,
+        ResolverClient,
     )
 except ImportError:
-    AssertionsClient = None
+    from datahub.sdk.resolver_client import (  # type: ignore[assignment]  # If the client is not installed, use the one from the SDK
+        ResolverClient,
+    )
 
 
 class DataHubClient:
@@ -112,9 +113,27 @@ class DataHubClient:
         return LineageClient(self)
 
     @property
-    def assertions(self) -> AssertionsClient:  # type: ignore[return-value]  # Type is not available if assertion_client is not installed
-        if AssertionsClient is None:
-            raise SdkUsageError(
-                "AssertionsClient is not installed, please install it with `pip install acryl-datahub-cloud`"
-            )
+    def assertions(self):  # type: ignore[report-untyped-call]  # Not available due to circular import issues
+        try:
+            from acryl_datahub_cloud._sdk_extras import AssertionsClient
+        except ImportError as e:
+            if "acryl_datahub_cloud" in str(e):
+                raise SdkUsageError(
+                    "AssertionsClient is not installed, please install it with `pip install acryl-datahub-cloud`"
+                ) from e
+            else:
+                raise e
         return AssertionsClient(self)
+
+    @property
+    def subscriptions(self):  # type: ignore[report-untyped-call]  # Not available due to circular import issues
+        try:
+            from acryl_datahub_cloud._sdk_extras import SubscriptionClient
+        except ImportError as e:
+            if "acryl_datahub_cloud" in str(e):
+                raise SdkUsageError(
+                    "SubscriptionClient is not installed, please install it with `pip install acryl-datahub-cloud`"
+                ) from e
+            else:
+                raise e
+        return SubscriptionClient(self)

datahub/sdk/mlmodel.py

@@ -24,6 +24,7 @@ from datahub.sdk._shared import (
     HasInstitutionalMemory,
     HasOwnership,
     HasPlatformInstance,
+    HasStructuredProperties,
     HasTags,
     HasTerms,
     HasVersion,
@@ -31,6 +32,7 @@ from datahub.sdk._shared import (
     LinksInputType,
     MLTrainingJobInputType,
     OwnersInputType,
+    StructuredPropertyInputType,
     TagsInputType,
     TermsInputType,
     TrainingMetricsInputType,
@@ -50,6 +52,7 @@ class MLModel(
     HasTerms,
     HasDomain,
     HasVersion,
+    HasStructuredProperties,
     Entity,
 ):
     __slots__ = ()
@@ -82,53 +85,43 @@ class MLModel(
         model_group: Optional[Union[str, MlModelGroupUrn]] = None,
         training_jobs: Optional[MLTrainingJobInputType] = None,
         downstream_jobs: Optional[MLTrainingJobInputType] = None,
+        structured_properties: Optional[StructuredPropertyInputType] = None,
         extra_aspects: ExtraAspectsType = None,
     ):
         urn = MlModelUrn(platform=platform, name=id, env=env)
         super().__init__(urn)
         self._set_extra_aspects(extra_aspects)
-
         self._set_platform_instance(urn.platform, platform_instance)
-
         self._ensure_model_props()
 
-        if version is not None:
-            self.set_version(version)
-        if name is not None:
-            self.set_name(name)
-        if aliases is not None:
-            self.set_version_aliases(aliases)
-        if description is not None:
-            self.set_description(description)
-        if training_metrics is not None:
-            self.set_training_metrics(training_metrics)
-        if hyper_params is not None:
-            self.set_hyper_params(hyper_params)
-        if external_url is not None:
-            self.set_external_url(external_url)
-        if custom_properties is not None:
-            self.set_custom_properties(custom_properties)
-        if created is not None:
-            self.set_created(created)
-        if last_modified is not None:
-            self.set_last_modified(last_modified)
+        # Initialize properties in logical groups
+        self._init_basic_properties(
+            version=version,
+            name=name,
+            aliases=aliases,
+            description=description,
+            external_url=external_url,
+            custom_properties=custom_properties,
+            created=created,
+            last_modified=last_modified,
+        )
 
-        if owners is not None:
-            self.set_owners(owners)
-        if links is not None:
-            self.set_links(links)
-        if tags is not None:
-            self.set_tags(tags)
-        if terms is not None:
-            self.set_terms(terms)
-        if domain is not None:
-            self.set_domain(domain)
-        if model_group is not None:
-            self.set_model_group(model_group)
-        if training_jobs is not None:
-            self.set_training_jobs(training_jobs)
-        if downstream_jobs is not None:
-            self.set_downstream_jobs(downstream_jobs)
+        self._init_ml_specific_properties(
+            training_metrics=training_metrics,
+            hyper_params=hyper_params,
+            model_group=model_group,
+            training_jobs=training_jobs,
+            downstream_jobs=downstream_jobs,
+        )
+
+        self._init_metadata_properties(
+            owners=owners,
+            links=links,
+            tags=tags,
+            terms=terms,
+            domain=domain,
+            structured_properties=structured_properties,
+        )
 
     @classmethod
     def _new_from_graph(cls, urn: Urn, current_aspects: AspectBag) -> Self:
@@ -299,3 +292,73 @@ class MLModel(
             props.downstreamJobs = [
                 job for job in props.downstreamJobs if job != job_str
             ]
+
+    def _init_basic_properties(
+        self,
+        version: Optional[str] = None,
+        name: Optional[str] = None,
+        aliases: Optional[List[str]] = None,
+        description: Optional[str] = None,
+        external_url: Optional[str] = None,
+        custom_properties: Optional[Dict[str, str]] = None,
+        created: Optional[datetime] = None,
+        last_modified: Optional[datetime] = None,
+    ) -> None:
+        if version is not None:
+            self.set_version(version)
+        if name is not None:
+            self.set_name(name)
+        if aliases is not None:
+            self.set_version_aliases(aliases)
+        if description is not None:
+            self.set_description(description)
+        if external_url is not None:
+            self.set_external_url(external_url)
+        if custom_properties is not None:
+            self.set_custom_properties(custom_properties)
+        if created is not None:
+            self.set_created(created)
+        if last_modified is not None:
+            self.set_last_modified(last_modified)
+
+    def _init_ml_specific_properties(
+        self,
+        training_metrics: Optional[TrainingMetricsInputType] = None,
+        hyper_params: Optional[HyperParamsInputType] = None,
+        model_group: Optional[Union[str, MlModelGroupUrn]] = None,
+        training_jobs: Optional[MLTrainingJobInputType] = None,
+        downstream_jobs: Optional[MLTrainingJobInputType] = None,
+    ) -> None:
+        if training_metrics is not None:
+            self.set_training_metrics(training_metrics)
+        if hyper_params is not None:
+            self.set_hyper_params(hyper_params)
+        if model_group is not None:
+            self.set_model_group(model_group)
+        if training_jobs is not None:
+            self.set_training_jobs(training_jobs)
+        if downstream_jobs is not None:
+            self.set_downstream_jobs(downstream_jobs)
+
+    def _init_metadata_properties(
+        self,
+        owners: Optional[OwnersInputType] = None,
+        links: Optional[LinksInputType] = None,
+        tags: Optional[TagsInputType] = None,
+        terms: Optional[TermsInputType] = None,
+        domain: Optional[DomainInputType] = None,
+        structured_properties: Optional[StructuredPropertyInputType] = None,
+    ) -> None:
+        if owners is not None:
+            self.set_owners(owners)
+        if links is not None:
+            self.set_links(links)
+        if tags is not None:
+            self.set_tags(tags)
+        if terms is not None:
+            self.set_terms(terms)
+        if domain is not None:
+            self.set_domain(domain)
+        if structured_properties is not None:
+            for key, value in structured_properties.items():
+                self.set_structured_property(property_urn=key, values=value)

datahub/sdk/mlmodelgroup.py

@@ -17,10 +17,12 @@ from datahub.sdk._shared import (
     HasInstitutionalMemory,
     HasOwnership,
     HasPlatformInstance,
+    HasStructuredProperties,
     HasTags,
     HasTerms,
     LinksInputType,
     OwnersInputType,
+    StructuredPropertyInputType,
     TagsInputType,
     TermsInputType,
     make_time_stamp,
@@ -36,6 +38,7 @@ class MLModelGroup(
     HasTags,
     HasTerms,
     HasDomain,
+    HasStructuredProperties,
     Entity,
 ):
     __slots__ = ()
@@ -66,6 +69,7 @@ class MLModelGroup(
         domain: Optional[DomainInputType] = None,
         training_jobs: Optional[Sequence[Union[str, DataProcessInstanceUrn]]] = None,
         downstream_jobs: Optional[Sequence[Union[str, DataProcessInstanceUrn]]] = None,
+        structured_properties: Optional[StructuredPropertyInputType] = None,
         extra_aspects: ExtraAspectsType = None,
     ):
         urn = MlModelGroupUrn(platform=platform, name=id, env=env)
@@ -105,6 +109,9 @@ class MLModelGroup(
             self.set_training_jobs(training_jobs)
         if downstream_jobs is not None:
             self.set_downstream_jobs(downstream_jobs)
+        if structured_properties is not None:
+            for key, value in structured_properties.items():
+                self.set_structured_property(property_urn=key, values=value)
 
     @classmethod
     def _new_from_graph(cls, urn: Urn, current_aspects: AspectBag) -> Self:

datahub/sql_parsing/sql_parsing_aggregator.py

@@ -109,7 +109,7 @@ class ObservedQuery:
     query_hash: Optional[str] = None
     usage_multiplier: int = 1
 
-    # Use this to store addtitional key-value information about query for debugging
+    # Use this to store additional key-value information about the query for debugging.
     extra_info: Optional[dict] = None
 
 

datahub/testing/mce_helpers.py

@@ -0,0 +1,421 @@
+import json
+import logging
+import os
+import pathlib
+import re
+import tempfile
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    Type,
+    Union,
+)
+
+import pytest
+
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.sink.file import write_metadata_file
+from datahub.metadata.schema_classes import MetadataChangeEventClass
+from datahub.metadata.urns import Urn
+from datahub.testing.compare_metadata_json import (
+    assert_metadata_files_equal,
+    load_json_file,
+)
+
+logger = logging.getLogger(__name__)
+
+IGNORE_PATH_TIMESTAMPS = [
+    # Ignore timestamps from the ETL pipeline. A couple examples:
+    r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['created'\]\['time'\]",
+    r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['lastModified'\]\['time'\]",
+    r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['createStamp'\]\['time'\]",
+    r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['auditStamp'\]\['time'\]",
+]
+
+
+class MCEConstants:
+    PROPOSED_SNAPSHOT = "proposedSnapshot"
+    DATASET_SNAPSHOT_CLASS = (
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot"
+    )
+
+
+class MCPConstants:
+    CHANGE_TYPE = "changeType"
+    ENTITY_URN = "entityUrn"
+    ENTITY_TYPE = "entityType"
+    ASPECT_NAME = "aspectName"
+    ASPECT_VALUE = "aspect"
+
+
+class EntityType:
+    DATASET = "dataset"
+    PIPELINE = "dataFlow"
+    FLOW = "dataFlow"
+    TASK = "dataJob"
+    JOB = "dataJob"
+    USER = "corpuser"
+    GROUP = "corpGroup"
+
+
+def clean_nones(value):
+    """
+    Recursively remove all None values from dictionaries and lists, and returns
+    the result as a new dictionary or list.
+    """
+    if isinstance(value, list):
+        return [clean_nones(x) for x in value if x is not None]
+    elif isinstance(value, dict):
+        return {key: clean_nones(val) for key, val in value.items() if val is not None}
+    else:
+        return value
+
+
+def check_golden_file(
+    pytestconfig: pytest.Config,
+    output_path: Union[str, os.PathLike],
+    golden_path: Union[str, os.PathLike],
+    ignore_paths: Sequence[str] = (),
+    ignore_paths_v2: Sequence[str] = (),
+    ignore_order: bool = True,
+) -> None:
+    # TODO: Remove the pytestconfig parameter since it's redundant.
+    # Or more straightforward - we can remove the `check_golden_file` method
+    # and use assert_metadata_files_equal directly. Maybe call it "check_golden_metadata"?
+    # In a lot of cases, the output_path is also just annoying - our pytest setup
+    # should be responsible for figuring out where to put the temp file.
+    assert_metadata_files_equal(
+        output_path=output_path,
+        golden_path=golden_path,
+        ignore_paths=ignore_paths,
+        ignore_paths_v2=ignore_paths_v2,
+        ignore_order=ignore_order,
+    )
+
+
+def check_goldens_stream(
+    outputs: List,
+    golden_path: Union[str, os.PathLike],
+    ignore_paths: Sequence[str] = (),
+    ignore_order: bool = True,
+) -> None:
+    with tempfile.NamedTemporaryFile() as f:
+        write_metadata_file(pathlib.Path(f.name), outputs)
+
+        assert_metadata_files_equal(
+            output_path=f.name,
+            golden_path=golden_path,
+            ignore_paths=ignore_paths,
+            ignore_order=ignore_order,
+        )
+
+
+def _get_field_for_entity_type_in_mce(entity_type: str) -> str:
+    """Returns the field to look for depending on the type of entity in the MCE"""
+    if entity_type == EntityType.DATASET:
+        return MCEConstants.DATASET_SNAPSHOT_CLASS
+    raise Exception(f"Not implemented for entity_type {entity_type}")
+
+
+def _get_filter(
+    mce: bool = False, mcp: bool = False, entity_type: Optional[str] = None
+) -> Callable[[Dict], bool]:
+    if mce:
+        # cheap way to determine if we are working with an MCE for the appropriate entity_type
+        if entity_type:
+            return (
+                lambda x: MCEConstants.PROPOSED_SNAPSHOT in x
+                and _get_field_for_entity_type_in_mce(str(entity_type))
+                in x[MCEConstants.PROPOSED_SNAPSHOT]
+            )
+        else:
+            return lambda x: MCEConstants.PROPOSED_SNAPSHOT in x
+    if mcp:
+        # cheap way to determine if we are working with an MCP
+        return lambda x: MCPConstants.CHANGE_TYPE in x and (
+            x[MCPConstants.ENTITY_TYPE] == entity_type if entity_type else True
+        )
+    return lambda _: False
+
+
+def _get_element(event: Dict[str, Any], path_spec: List[str]) -> Any:
+    try:
+        for p in path_spec:
+            if p not in event:
+                return None
+            else:
+                event = event.get(p, {})
+        return event
+    except Exception as e:
+        print(event)
+        raise e
+
+
+def _element_matches_pattern(
+    event: Dict[str, Any], path_spec: List[str], pattern: str
+) -> Tuple[bool, bool]:
+    import re
+
+    element = _get_element(event, path_spec)
+    if element is None:
+        return (False, False)
+    else:
+        return (True, re.search(pattern, str(element)) is not None)
+
+
+def get_entity_urns(events_file: str) -> Set[str]:
+    events = load_json_file(events_file)
+    assert isinstance(events, list)
+    return _get_entity_urns(events)
+
+
+def _get_entity_urns(events_list: List[Dict]) -> Set[str]:
+    entity_type = "dataset"
+    # mce urns
+    mce_urns = {
+        _get_element(x, _get_mce_urn_path_spec(entity_type))
+        for x in events_list
+        if _get_filter(mce=True, entity_type=entity_type)(x)
+    }
+    mcp_urns = {
+        _get_element(x, _get_mcp_urn_path_spec())
+        for x in events_list
+        if _get_filter(mcp=True, entity_type=entity_type)(x)
+    }
+    all_urns = mce_urns.union(mcp_urns)
+    return all_urns
+
+
+def assert_mcp_entity_urn(
+    filter: str, entity_type: str, regex_pattern: str, file: str
+) -> int:
+    def get_path_spec_for_urn() -> List[str]:
+        return [MCPConstants.ENTITY_URN]
+
+    test_output = load_json_file(file)
+    if isinstance(test_output, list):
+        path_spec = get_path_spec_for_urn()
+        filter_operator = _get_filter(mcp=True, entity_type=entity_type)
+        filtered_events = [
+            (x, _element_matches_pattern(x, path_spec, regex_pattern))
+            for x in test_output
+            if filter_operator(x)
+        ]
+        failed_events = [y for y in filtered_events if not y[1][0] or not y[1][1]]
+        if failed_events:
+            raise Exception("Failed to match events", failed_events)
+        return len(filtered_events)
+    else:
+        raise Exception(
+            f"Did not expect the file {file} to not contain a list of items"
+        )
+
+
+def _get_mce_urn_path_spec(entity_type: str) -> List[str]:
+    if entity_type == EntityType.DATASET:
+        return [
+            MCEConstants.PROPOSED_SNAPSHOT,
+            MCEConstants.DATASET_SNAPSHOT_CLASS,
+            "urn",
+        ]
+    raise Exception(f"Not implemented for entity_type: {entity_type}")
+
+
+def _get_mcp_urn_path_spec() -> List[str]:
+    return [MCPConstants.ENTITY_URN]
+
+
+def assert_mce_entity_urn(
+    filter: str, entity_type: str, regex_pattern: str, file: str
+) -> int:
+    """Assert that all mce entity urns must match the regex pattern passed in. Return the number of events matched"""
+
+    test_output = load_json_file(file)
+    if isinstance(test_output, list):
+        path_spec = _get_mce_urn_path_spec(entity_type)
+        filter_operator = _get_filter(mce=True)
+        filtered_events = [
+            (x, _element_matches_pattern(x, path_spec, regex_pattern))
+            for x in test_output
+            if filter_operator(x)
+        ]
+        failed_events = [y for y in filtered_events if not y[1][0] or not y[1][1]]
+        if failed_events:
+            raise Exception(
+                "Failed to match events: {json.dumps(failed_events, indent=2)}"
+            )
+        return len(filtered_events)
+    else:
+        raise Exception(
+            f"Did not expect the file {file} to not contain a list of items"
+        )
+
+
+def assert_for_each_entity(
+    entity_type: str,
+    aspect_name: str,
+    aspect_field_matcher: Dict[str, Any],
+    file: str,
+    exception_urns: Optional[List[str]] = None,
+) -> int:
+    """Assert that an aspect name with the desired fields exists for each entity urn"""
+    if exception_urns is None:
+        exception_urns = []
+    test_output = load_json_file(file)
+    assert isinstance(test_output, list)
+    # mce urns
+    mce_urns = {
+        _get_element(x, _get_mce_urn_path_spec(entity_type))
+        for x in test_output
+        if _get_filter(mce=True, entity_type=entity_type)(x)
+    }
+    mcp_urns = {
+        _get_element(x, _get_mcp_urn_path_spec())
+        for x in test_output
+        if _get_filter(mcp=True, entity_type=entity_type)(x)
+    }
+    all_urns = mce_urns.union(mcp_urns)
+    # there should not be any None urns
+    assert None not in all_urns
+    aspect_map = {urn: None for urn in all_urns}
+    # iterate over all mcps
+    for o in [
+        mcp
+        for mcp in test_output
+        if _get_filter(mcp=True, entity_type=entity_type)(mcp)
+    ]:
+        if o.get(MCPConstants.ASPECT_NAME) == aspect_name:
+            # load the inner aspect payload and assign to this urn
+            aspect_map[o[MCPConstants.ENTITY_URN]] = o.get(
+                MCPConstants.ASPECT_VALUE, {}
+            ).get("json")
+
+    success: List[str] = []
+    failures: List[str] = []
+    for urn, aspect_val in aspect_map.items():
+        if aspect_val is not None:
+            for f in aspect_field_matcher:
+                assert aspect_field_matcher[f] == _get_element(aspect_val, [f]), (
+                    f"urn: {urn} -> Field {f} must match value {aspect_field_matcher[f]}, found {_get_element(aspect_val, [f])}"
+                )
+            success.append(urn)
+        elif urn not in exception_urns:
+            print(f"Adding {urn} to failures")
+            failures.append(urn)
+
+    if success:
+        print(f"Succeeded on assertion for urns {success}")
+    if failures:
+        raise AssertionError(
+            f"Failed to find aspect_name {aspect_name} for urns {json.dumps(failures, indent=2)}"
+        )
+
+    return len(success)
+
+
+def assert_entity_mce_aspect(
+    entity_urn: str, aspect: Any, aspect_type: Type, file: str
+) -> int:
+    # TODO: Replace with read_metadata_file()
+    test_output = load_json_file(file)
+    entity_type = Urn.from_string(entity_urn).entity_type
+    assert isinstance(test_output, list)
+    # mce urns
+    mces: List[MetadataChangeEventClass] = [
+        MetadataChangeEventClass.from_obj(x)
+        for x in test_output
+        if _get_filter(mce=True, entity_type=entity_type)(x)
+        and _get_element(x, _get_mce_urn_path_spec(entity_type)) == entity_urn
+    ]
+    matches = 0
+    for mce in mces:
+        for a in mce.proposedSnapshot.aspects:
+            if isinstance(a, aspect_type):
+                assert a == aspect
+                matches = matches + 1
+    return matches
+
+
+def assert_entity_mcp_aspect(
+    entity_urn: str, aspect_field_matcher: Dict[str, Any], aspect_name: str, file: str
+) -> int:
+    # TODO: Replace with read_metadata_file()
+    test_output = load_json_file(file)
+    entity_type = Urn.from_string(entity_urn).entity_type
+    assert isinstance(test_output, list)
+    # mcps that match entity_urn
+    mcps: List[MetadataChangeProposalWrapper] = [
+        MetadataChangeProposalWrapper.from_obj_require_wrapper(x)
+        for x in test_output
+        if _get_filter(mcp=True, entity_type=entity_type)(x)
+        and _get_element(x, _get_mcp_urn_path_spec()) == entity_urn
+    ]
+    matches = 0
+    for mcp in mcps:
+        if mcp.aspectName == aspect_name:
+            assert mcp.aspect
+            aspect_val = mcp.aspect.to_obj()
+            for f in aspect_field_matcher:
+                assert aspect_field_matcher[f] == _get_element(aspect_val, [f]), (
+                    f"urn: {mcp.entityUrn} -> Field {f} must match value {aspect_field_matcher[f]}, found {_get_element(aspect_val, [f])}"
+                )
+                matches = matches + 1
+    return matches
+
+
+def assert_entity_urn_not_like(entity_type: str, regex_pattern: str, file: str) -> int:
+    """Assert that there are no entity urns that match the regex pattern passed in. Returns the total number of events in the file"""
+
+    # TODO: Refactor common code with assert_entity_urn_like.
+    test_output = load_json_file(file)
+    assert isinstance(test_output, list)
+    # mce urns
+    mce_urns = {
+        _get_element(x, _get_mce_urn_path_spec(entity_type))
+        for x in test_output
+        if _get_filter(mce=True, entity_type=entity_type)(x)
+    }
+    mcp_urns = {
+        _get_element(x, _get_mcp_urn_path_spec())
+        for x in test_output
+        if _get_filter(mcp=True, entity_type=entity_type)(x)
+    }
+    all_urns = mce_urns.union(mcp_urns)
+    print(all_urns)
+    matched_urns = [u for u in all_urns if re.match(regex_pattern, u)]
+    if matched_urns:
+        raise AssertionError(f"urns found that match the deny list {matched_urns}")
+    return len(test_output)
+
+
+def assert_entity_urn_like(entity_type: str, regex_pattern: str, file: str) -> int:
+    """Assert that there exist entity urns that match the regex pattern passed in. Returns the total number of events in the file"""
+
+    test_output = load_json_file(file)
+    assert isinstance(test_output, list)
+    # mce urns
+    mce_urns = {
+        _get_element(x, _get_mce_urn_path_spec(entity_type))
+        for x in test_output
+        if _get_filter(mce=True, entity_type=entity_type)(x)
+    }
+    mcp_urns = {
+        _get_element(x, _get_mcp_urn_path_spec())
+        for x in test_output
+        if _get_filter(mcp=True, entity_type=entity_type)(x)
+    }
+    all_urns = mce_urns.union(mcp_urns)
+    print(all_urns)
+    matched_urns = [u for u in all_urns if re.match(regex_pattern, u)]
+    if matched_urns:
+        return len(matched_urns)
+    else:
+        raise AssertionError(
+            f"No urns found that match the pattern {regex_pattern}. Full list is {all_urns}"
+        )