acryl-datahub-cloud 0.3.8rc17__tar.gz → 0.3.8.1rc1__tar.gz

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/PKG-INFO

@@ -1,90 +1,90 @@
 Metadata-Version: 2.2
 Name: acryl-datahub-cloud
-Version: 0.3.8rc17
+Version: 0.3.8.1rc1
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
 Provides-Extra: datahub-lineage-features
-Requires-Dist: pyarrow; extra == "datahub-lineage-features"
 Requires-Dist: duckdb; extra == "datahub-lineage-features"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Requires-Dist: pandas; extra == "datahub-lineage-features"
+Requires-Dist: pyarrow; extra == "datahub-lineage-features"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
-Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
 Requires-Dist: duckdb; extra == "datahub-reporting-forms"
+Requires-Dist: boto3; extra == "datahub-reporting-forms"
+Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Requires-Dist: pandas; extra == "datahub-reporting-forms"
-Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
-Requires-Dist: boto3; extra == "datahub-reporting-forms"
+Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
-Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
-Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-reporting
 Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-reporting"
-Requires-Dist: duckdb; extra == "datahub-usage-reporting"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
-Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
 Requires-Dist: pandas; extra == "datahub-usage-reporting"
-Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
+Requires-Dist: polars==1.19.0; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
+Requires-Dist: duckdb; extra == "datahub-usage-reporting"
 Requires-Dist: boto3; extra == "datahub-usage-reporting"
-Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
 Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
-Requires-Dist: pydantic<2; extra == "datahub-usage-reporting"
 Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
-Requires-Dist: polars==1.19.0; extra == "datahub-usage-reporting"
+Requires-Dist: pydantic<2; extra == "datahub-usage-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
+Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
 Provides-Extra: datahub-metadata-sharing
 Requires-Dist: tenacity; extra == "datahub-metadata-sharing"
 Provides-Extra: acryl-cs-issues
 Requires-Dist: jinja2; extra == "acryl-cs-issues"
-Requires-Dist: zenpy; extra == "acryl-cs-issues"
 Requires-Dist: openai; extra == "acryl-cs-issues"
+Requires-Dist: zenpy; extra == "acryl-cs-issues"
 Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
 Provides-Extra: all
+Requires-Dist: jinja2; extra == "all"
 Requires-Dist: elasticsearch==7.13.4; extra == "all"
-Requires-Dist: opensearch-py==2.4.2; extra == "all"
 Requires-Dist: pandas; extra == "all"
+Requires-Dist: polars==1.19.0; extra == "all"
+Requires-Dist: opensearch-py==2.4.2; extra == "all"
+Requires-Dist: duckdb; extra == "all"
+Requires-Dist: pydantic<2; extra == "all"
 Requires-Dist: scipy<=1.14.1; extra == "all"
-Requires-Dist: tenacity; extra == "all"
-Requires-Dist: openai; extra == "all"
 Requires-Dist: zenpy; extra == "all"
-Requires-Dist: pyarrow<=18.0.0; extra == "all"
-Requires-Dist: pydantic<2; extra == "all"
-Requires-Dist: termcolor==2.5.0; extra == "all"
-Requires-Dist: duckdb; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: boto3; extra == "all"
-Requires-Dist: jinja2; extra == "all"
 Requires-Dist: numpy<2; extra == "all"
 Requires-Dist: slack-sdk; extra == "all"
-Requires-Dist: polars==1.19.0; extra == "all"
+Requires-Dist: tenacity; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: pyarrow<=18.0.0; extra == "all"
+Requires-Dist: boto3; extra == "all"
+Requires-Dist: termcolor==2.5.0; extra == "all"
+Requires-Dist: openai; extra == "all"
 Provides-Extra: dev
-Requires-Dist: elasticsearch==7.13.4; extra == "dev"
-Requires-Dist: duckdb; extra == "dev"
-Requires-Dist: opensearch-py==2.4.2; extra == "dev"
-Requires-Dist: pyarrow; extra == "dev"
 Requires-Dist: acryl-datahub[dev]; extra == "dev"
+Requires-Dist: elasticsearch==7.13.4; extra == "dev"
+Requires-Dist: jinja2; extra == "dev"
 Requires-Dist: pandas; extra == "dev"
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: boto3; extra == "dev"
+Requires-Dist: polars==1.19.0; extra == "dev"
 Requires-Dist: tenacity; extra == "dev"
-Requires-Dist: openai; extra == "dev"
 Requires-Dist: slack-sdk; extra == "dev"
-Requires-Dist: jinja2; extra == "dev"
-Requires-Dist: zenpy; extra == "dev"
-Requires-Dist: numpy<2; extra == "dev"
+Requires-Dist: pyarrow; extra == "dev"
+Requires-Dist: opensearch-py==2.4.2; extra == "dev"
+Requires-Dist: duckdb; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
 Requires-Dist: pyarrow<=18.0.0; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
 Requires-Dist: termcolor==2.5.0; extra == "dev"
-Requires-Dist: polars==1.19.0; extra == "dev"
+Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
+Requires-Dist: openai; extra == "dev"
+Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: numpy<2; extra == "dev"
 Dynamic: provides-extra
 Dynamic: requires-dist

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.8rc17",
+  "version": "0.3.8.1rc1",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/datahub_metadata_sharing/metadata_sharing_source.py

@@ -104,9 +104,8 @@ class DataHubMetadataSharingSource(Source):
                     self._process_single_entity(result)
 
                 self.report.batches_processed = current_batch_number
-                self.report.info(
-                    message="Completed sharing batch of entities.",
-                    context=f"{current_batch_number} of size {self.config.batch_size}!",
+                logger.info(
+                    f"Completed sharing batch of entities {current_batch_number} of size {self.config.batch_size}!"
                 )
                 current_batch_number += 1
 
@@ -126,10 +125,8 @@ class DataHubMetadataSharingSource(Source):
             )
             return
 
-        self.report.info(
-            message="Completed sharing all entities.",
-            context=f"Successfully shared {self.report.entities_shared} entities, "
-            f"failed to share {self.report.entities_failed} entities.",
+        logger.info(
+            f"Successfully shared {self.report.entities_shared} entities, failed to share {self.report.entities_failed} entities.",
         )
 
     # Rest of the methods remain the same...
@@ -168,9 +165,9 @@ class DataHubMetadataSharingSource(Source):
                     continue
 
                 if previous_status != "SUCCESS":
-                    self.report.report_warning(
-                        message="Attempting to share a previously unsuccessful shared entity!",
-                        context=f"entity urn: {entity_urn}, destination urn: {destination_urn}",
+                    logger.info(
+                        "Attempting to share a previously unsuccessful shared entity! "
+                        + f"entity urn: {entity_urn}, destination urn: {destination_urn}"
                     )
 
                 lineage_direction = self._determine_lineage_direction(share_config)

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/datahub_reporting/forms.py

@@ -93,7 +93,7 @@ class DataHubReportingFormsSource(Source):
         form_analytics_config = self.get_reporting_config()
 
         if form_analytics_config and not form_analytics_config.enabled:
-            logger.error("Form analytics is not enabled. Skipping reporting.")
+            logger.info("Form analytics is not enabled. Skipping reporting.")
             self.report.feature_enabled = False
             return
 

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py

@@ -1377,7 +1377,7 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
 
         top_users = self.generate_top_users(lf)
 
-        usage_with_top_users = top_users.join(total_queries, on="urn", how="inner")
+        usage_with_top_users = total_queries.join(top_users, on="urn", how="left")
 
         usage_with_top_users_with_ranks = self.gen_rank_and_percentile(
             usage_with_top_users, "totalSqlQueries", "urn", "platform", "queries_"

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/metadata/_urns/urn_defs.py

@@ -842,6 +842,51 @@ class IncidentUrn(_SpecificUrn):
     def id(self) -> str:
         return self.entity_ids[0]
 
+if TYPE_CHECKING:
+    from datahub.metadata.schema_classes import RemoteExecutorPoolKeyClass
+
+class DataHubRemoteExecutorPoolUrn(_SpecificUrn):
+    ENTITY_TYPE: ClassVar[str] = "dataHubRemoteExecutorPool"
+    URN_PARTS: ClassVar[int] = 1
+
+    def __init__(self, name: str, *, _allow_coercion: bool = True) -> None:
+        if _allow_coercion:
+            # Field coercion logic (if any is required).
+            name = UrnEncoder.encode_string(name)
+
+        # Validation logic.
+        if not name:
+            raise InvalidUrnError("DataHubRemoteExecutorPoolUrn name cannot be empty")
+        if UrnEncoder.contains_reserved_char(name):
+            raise InvalidUrnError(f'DataHubRemoteExecutorPoolUrn name contains reserved characters')
+
+        super().__init__(self.ENTITY_TYPE, [name])
+
+    @classmethod
+    def _parse_ids(cls, entity_ids: List[str]) -> "DataHubRemoteExecutorPoolUrn":
+        if len(entity_ids) != cls.URN_PARTS:
+            raise InvalidUrnError(f"DataHubRemoteExecutorPoolUrn should have {cls.URN_PARTS} parts, got {len(entity_ids)}: {entity_ids}")
+        return cls(name=entity_ids[0], _allow_coercion=False)
+
+    @classmethod
+    def underlying_key_aspect_type(cls) -> Type["RemoteExecutorPoolKeyClass"]:
+        from datahub.metadata.schema_classes import RemoteExecutorPoolKeyClass
+
+        return RemoteExecutorPoolKeyClass
+
+    def to_key_aspect(self) -> "RemoteExecutorPoolKeyClass":
+        from datahub.metadata.schema_classes import RemoteExecutorPoolKeyClass
+
+        return RemoteExecutorPoolKeyClass(name=self.name)
+
+    @classmethod
+    def from_key_aspect(cls, key_aspect: "RemoteExecutorPoolKeyClass") -> "DataHubRemoteExecutorPoolUrn":
+        return cls(name=key_aspect.name)
+
+    @property
+    def name(self) -> str:
+        return self.entity_ids[0]
+
 if TYPE_CHECKING:
     from datahub.metadata.schema_classes import TelemetryKeyClass
 

acryl_datahub_cloud-0.3.8.1rc1/src/acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/executorpool/__init__.py

@@ -0,0 +1,15 @@
+# mypy: ignore-errors
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# pylint: skip-file
+# fmt: off
+# isort: skip_file
+from .....schema_classes import RemoteExecutorPoolInfoClass
+
+
+RemoteExecutorPoolInfo = RemoteExecutorPoolInfoClass
+
+# fmt: on

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py

@@ -60,6 +60,7 @@ from ......schema_classes import PostKeyClass
 from ......schema_classes import QueryKeyClass
 from ......schema_classes import RecommendationModuleKeyClass
 from ......schema_classes import RemoteExecutorKeyClass
+from ......schema_classes import RemoteExecutorPoolKeyClass
 from ......schema_classes import RoleKeyClass
 from ......schema_classes import SchemaFieldKeyClass
 from ......schema_classes import SubscriptionKeyClass
@@ -122,6 +123,7 @@ PostKey = PostKeyClass
 QueryKey = QueryKeyClass
 RecommendationModuleKey = RecommendationModuleKeyClass
 RemoteExecutorKey = RemoteExecutorKeyClass
+RemoteExecutorPoolKey = RemoteExecutorPoolKeyClass
 RoleKey = RoleKeyClass
 SchemaFieldKey = SchemaFieldKeyClass
 SubscriptionKey = SubscriptionKeyClass

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/metadata/schema.avsc

@@ -9218,8 +9218,8 @@
           "fieldType": "KEYWORD"
         },
         "type": "string",
-        "name": "executorId",
-        "doc": "TODO: rename to pipeline id"
+        "name": "poolName",
+        "doc": "References the 'name' defined in RemoteExecutorPoolKey"
       },
       {
         "type": "string",
@@ -9279,7 +9279,7 @@
           "fieldType": "COUNT",
           "queryByDefault": false
         },
-        "type": "int",
+        "type": "long",
         "name": "reportedAt",
         "doc": "UTC-based timestamp of the last reported status"
       }
@@ -15779,6 +15779,30 @@
     ],
     "doc": "Key for an asset Incident"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataHubRemoteExecutorPoolKey",
+      "keyForEntity": "dataHubRemoteExecutorPool",
+      "entityCategory": "internal",
+      "entityAspects": [
+        "dataHubRemoteExecutorPoolInfo"
+      ]
+    },
+    "name": "RemoteExecutorPoolKey",
+    "namespace": "com.linkedin.pegasus2avro.metadata.key",
+    "fields": [
+      {
+        "Searchable": {
+          "fieldType": "KEYWORD"
+        },
+        "type": "string",
+        "name": "name",
+        "doc": "The unique identifier for the remote executor pool"
+      }
+    ],
+    "doc": "Key for an DataHub Remote Executor Pool"
+  },
   {
     "type": "record",
     "Aspect": {
@@ -22029,6 +22053,25 @@
     ],
     "doc": "Information about a connection to an external platform."
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataHubRemoteExecutorPoolInfo"
+    },
+    "name": "RemoteExecutorPoolInfo",
+    "namespace": "com.linkedin.pegasus2avro.executorpool",
+    "fields": [
+      {
+        "Searchable": {
+          "fieldType": "COUNT",
+          "queryByDefault": false
+        },
+        "type": "long",
+        "name": "createdAt",
+        "doc": "UTC-based timestamp of when this pool was created"
+      }
+    ]
+  },
   {
     "type": "record",
     "Aspect": {

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/metadata/schema_classes.py

@@ -17279,7 +17279,7 @@ class RemoteExecutorStatusClass(_Aspect):
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.executor.RemoteExecutorStatus")
 
     def __init__(self,
-        executorId: str,
+        poolName: str,
         executorReleaseVersion: str,
         executorAddress: str,
         executorHostname: str,
@@ -17293,7 +17293,7 @@ class RemoteExecutorStatusClass(_Aspect):
     ):
         super().__init__()
         
-        self.executorId = executorId
+        self.poolName = poolName
         self.executorReleaseVersion = executorReleaseVersion
         self.executorAddress = executorAddress
         self.executorHostname = executorHostname
@@ -17326,7 +17326,7 @@ class RemoteExecutorStatusClass(_Aspect):
         self.reportedAt = reportedAt
     
     def _restore_defaults(self) -> None:
-        self.executorId = str()
+        self.poolName = str()
         self.executorReleaseVersion = str()
         self.executorAddress = str()
         self.executorHostname = str()
@@ -17340,13 +17340,13 @@ class RemoteExecutorStatusClass(_Aspect):
     
     
     @property
-    def executorId(self) -> str:
-        """TODO: rename to pipeline id"""
-        return self._inner_dict.get('executorId')  # type: ignore
+    def poolName(self) -> str:
+        """References the 'name' defined in RemoteExecutorPoolKey"""
+        return self._inner_dict.get('poolName')  # type: ignore
     
-    @executorId.setter
-    def executorId(self, value: str) -> None:
-        self._inner_dict['executorId'] = value
+    @poolName.setter
+    def poolName(self, value: str) -> None:
+        self._inner_dict['poolName'] = value
     
     
     @property
@@ -17450,6 +17450,35 @@ class RemoteExecutorStatusClass(_Aspect):
         self._inner_dict['reportedAt'] = value
     
     
+class RemoteExecutorPoolInfoClass(_Aspect):
+    # No docs available.
+
+
+    ASPECT_NAME = 'dataHubRemoteExecutorPoolInfo'
+    ASPECT_INFO = {}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.executorpool.RemoteExecutorPoolInfo")
+
+    def __init__(self,
+        createdAt: int,
+    ):
+        super().__init__()
+        
+        self.createdAt = createdAt
+    
+    def _restore_defaults(self) -> None:
+        self.createdAt = int()
+    
+    
+    @property
+    def createdAt(self) -> int:
+        """UTC-based timestamp of when this pool was created"""
+        return self._inner_dict.get('createdAt')  # type: ignore
+    
+    @createdAt.setter
+    def createdAt(self, value: int) -> None:
+        self._inner_dict['createdAt'] = value
+    
+    
 class DomainParamsClass(DictWrapper):
     """Information specific to glossary terms prompts."""
     
@@ -22718,6 +22747,35 @@ class RemoteExecutorKeyClass(_Aspect):
         self._inner_dict['id'] = value
     
     
+class RemoteExecutorPoolKeyClass(_Aspect):
+    """Key for an DataHub Remote Executor Pool"""
+
+
+    ASPECT_NAME = 'dataHubRemoteExecutorPoolKey'
+    ASPECT_INFO = {'keyForEntity': 'dataHubRemoteExecutorPool', 'entityCategory': 'internal', 'entityAspects': ['dataHubRemoteExecutorPoolInfo']}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.RemoteExecutorPoolKey")
+
+    def __init__(self,
+        name: str,
+    ):
+        super().__init__()
+        
+        self.name = name
+    
+    def _restore_defaults(self) -> None:
+        self.name = str()
+    
+    
+    @property
+    def name(self) -> str:
+        """The unique identifier for the remote executor pool"""
+        return self._inner_dict.get('name')  # type: ignore
+    
+    @name.setter
+    def name(self, value: str) -> None:
+        self._inner_dict['name'] = value
+    
+    
 class RoleKeyClass(_Aspect):
     """Key for a External AccessManagement"""
 
@@ -35402,6 +35460,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.execution.ExecutionRequestSource': ExecutionRequestSourceClass,
     'com.linkedin.pegasus2avro.execution.StructuredExecutionReport': StructuredExecutionReportClass,
     'com.linkedin.pegasus2avro.executor.RemoteExecutorStatus': RemoteExecutorStatusClass,
+    'com.linkedin.pegasus2avro.executorpool.RemoteExecutorPoolInfo': RemoteExecutorPoolInfoClass,
     'com.linkedin.pegasus2avro.form.DomainParams': DomainParamsClass,
     'com.linkedin.pegasus2avro.form.DynamicFormAssignment': DynamicFormAssignmentClass,
     'com.linkedin.pegasus2avro.form.FormActorAssignment': FormActorAssignmentClass,
@@ -35519,6 +35578,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.metadata.key.QueryKey': QueryKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.RecommendationModuleKey': RecommendationModuleKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.RemoteExecutorKey': RemoteExecutorKeyClass,
+    'com.linkedin.pegasus2avro.metadata.key.RemoteExecutorPoolKey': RemoteExecutorPoolKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.RoleKey': RoleKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.SchemaFieldKey': SchemaFieldKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.SubscriptionKey': SubscriptionKeyClass,
@@ -36071,6 +36131,7 @@ __SCHEMA_TYPES = {
     'ExecutionRequestSource': ExecutionRequestSourceClass,
     'StructuredExecutionReport': StructuredExecutionReportClass,
     'RemoteExecutorStatus': RemoteExecutorStatusClass,
+    'RemoteExecutorPoolInfo': RemoteExecutorPoolInfoClass,
     'DomainParams': DomainParamsClass,
     'DynamicFormAssignment': DynamicFormAssignmentClass,
     'FormActorAssignment': FormActorAssignmentClass,
@@ -36188,6 +36249,7 @@ __SCHEMA_TYPES = {
     'QueryKey': QueryKeyClass,
     'RecommendationModuleKey': RecommendationModuleKeyClass,
     'RemoteExecutorKey': RemoteExecutorKeyClass,
+    'RemoteExecutorPoolKey': RemoteExecutorPoolKeyClass,
     'RoleKey': RoleKeyClass,
     'SchemaFieldKey': SchemaFieldKeyClass,
     'SubscriptionKey': SubscriptionKeyClass,
@@ -36574,6 +36636,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     MLModelGroupKeyClass,
     ERModelRelationshipKeyClass,
     IncidentKeyClass,
+    RemoteExecutorPoolKeyClass,
     TelemetryKeyClass,
     SchemaFieldKeyClass,
     LinkPreviewKeyClass,
@@ -36665,6 +36728,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     GroupMembershipClass,
     DataHubRetentionConfigClass,
     DataHubConnectionDetailsClass,
+    RemoteExecutorPoolInfoClass,
     AiInferenceMetadataClass,
     EntityInferenceMetadataClass,
     BusinessAttributesClass,
@@ -36841,6 +36905,7 @@ class AspectBag(TypedDict, total=False):
     mlModelGroupKey: MLModelGroupKeyClass
     erModelRelationshipKey: ERModelRelationshipKeyClass
     incidentKey: IncidentKeyClass
+    dataHubRemoteExecutorPoolKey: RemoteExecutorPoolKeyClass
     telemetryKey: TelemetryKeyClass
     schemaFieldKey: SchemaFieldKeyClass
     linkPreviewKey: LinkPreviewKeyClass
@@ -36932,6 +36997,7 @@ class AspectBag(TypedDict, total=False):
     groupMembership: GroupMembershipClass
     dataHubRetentionConfig: DataHubRetentionConfigClass
     dataHubConnectionDetails: DataHubConnectionDetailsClass
+    dataHubRemoteExecutorPoolInfo: RemoteExecutorPoolInfoClass
     aiInferenceMetadata: AiInferenceMetadataClass
     entityInferenceMetadata: EntityInferenceMetadataClass
     businessAttributes: BusinessAttributesClass
@@ -36990,6 +37056,7 @@ KEY_ASPECTS: Dict[str, Type[_Aspect]] = {
     'mlModelGroup': MLModelGroupKeyClass,
     'erModelRelationship': ERModelRelationshipKeyClass,
     'incident': IncidentKeyClass,
+    'dataHubRemoteExecutorPool': RemoteExecutorPoolKeyClass,
     'telemetry': TelemetryKeyClass,
     'schemaField': SchemaFieldKeyClass,
     'linkPreview': LinkPreviewKeyClass,

acryl_datahub_cloud-0.3.8.1rc1/src/acryl_datahub_cloud/metadata/schemas/RemoteExecutorPoolInfo.avsc

@@ -0,0 +1,19 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubRemoteExecutorPoolInfo"
+  },
+  "name": "RemoteExecutorPoolInfo",
+  "namespace": "com.linkedin.pegasus2avro.executorpool",
+  "fields": [
+    {
+      "Searchable": {
+        "fieldType": "COUNT",
+        "queryByDefault": false
+      },
+      "type": "long",
+      "name": "createdAt",
+      "doc": "UTC-based timestamp of when this pool was created"
+    }
+  ]
+}

acryl_datahub_cloud-0.3.8.1rc1/src/acryl_datahub_cloud/metadata/schemas/RemoteExecutorPoolKey.avsc

@@ -0,0 +1,24 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubRemoteExecutorPoolKey",
+    "keyForEntity": "dataHubRemoteExecutorPool",
+    "entityCategory": "internal",
+    "entityAspects": [
+      "dataHubRemoteExecutorPoolInfo"
+    ]
+  },
+  "name": "RemoteExecutorPoolKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "type": "string",
+      "name": "name",
+      "doc": "The unique identifier for the remote executor pool"
+    }
+  ],
+  "doc": "Key for an DataHub Remote Executor Pool"
+}

{acryl_datahub_cloud-0.3.8rc17 → acryl_datahub_cloud-0.3.8.1rc1}/src/acryl_datahub_cloud/metadata/schemas/RemoteExecutorStatus.avsc

@@ -11,8 +11,8 @@
         "fieldType": "KEYWORD"
       },
       "type": "string",
-      "name": "executorId",
-      "doc": "TODO: rename to pipeline id"
+      "name": "poolName",
+      "doc": "References the 'name' defined in RemoteExecutorPoolKey"
     },
     {
       "type": "string",
@@ -72,7 +72,7 @@
         "fieldType": "COUNT",
         "queryByDefault": false
       },
-      "type": "int",
+      "type": "long",
       "name": "reportedAt",
       "doc": "UTC-based timestamp of the last reported status"
     }