acryl-datahub 1.0.0rc18__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

datahub/ingestion/source/sac/sac.py

@@ -178,7 +178,9 @@ class SACSourceReport(StaleEntityRemovalSourceReport):
     SourceCapability.LINEAGE_COARSE,
     "Enabled by default (only for Live Data Models)",
 )
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 @capability(
     SourceCapability.SCHEMA_METADATA,
     "Enabled by default (only for Import Data Models)",

datahub/ingestion/source/salesforce.py

@@ -33,7 +33,10 @@ from datahub.ingestion.api.decorators import (
 )
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.common.subtypes import DatasetSubTypes
+from datahub.ingestion.source.common.subtypes import (
+    DatasetSubTypes,
+    SourceCapabilityModifier,
+)
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
     StaleEntityRemovalSourceReport,
@@ -107,30 +110,33 @@ class SalesforceConfig(
     auth: SalesforceAuthType = SalesforceAuthType.USERNAME_PASSWORD
 
     # Username, Password Auth
-    username: Optional[str] = Field(description="Salesforce username")
-    password: Optional[str] = Field(description="Password for Salesforce user")
+    username: Optional[str] = Field(None, description="Salesforce username")
+    password: Optional[str] = Field(None, description="Password for Salesforce user")
     consumer_key: Optional[str] = Field(
-        description="Consumer key for Salesforce JSON web token access"
+        None, description="Consumer key for Salesforce JSON web token access"
     )
     private_key: Optional[str] = Field(
-        description="Private key as a string for Salesforce JSON web token access"
+        None, description="Private key as a string for Salesforce JSON web token access"
     )
     security_token: Optional[str] = Field(
-        description="Security token for Salesforce username"
+        None, description="Security token for Salesforce username"
     )
     # client_id, client_secret not required
 
     # Direct - Instance URL, Access Token Auth
     instance_url: Optional[str] = Field(
-        description="Salesforce instance url. e.g. https://MyDomainName.my.salesforce.com"
+        None,
+        description="Salesforce instance url. e.g. https://MyDomainName.my.salesforce.com",
     )
     # Flag to indicate whether the instance is production or sandbox
     is_sandbox: bool = Field(
         default=False, description="Connect to Sandbox instance of your Salesforce"
     )
-    access_token: Optional[str] = Field(description="Access token for instance url")
+    access_token: Optional[str] = Field(
+        None, description="Access token for instance url"
+    )
 
-    ingest_tags: Optional[bool] = Field(
+    ingest_tags: bool = Field(
         default=False,
         description="Ingest Tags from source. This will override Tags entered from UI",
     )
@@ -144,7 +150,8 @@ class SalesforceConfig(
         description='Regex patterns for tables/schemas to describe domain_key domain key (domain_key can be any string like "sales".) There can be multiple domain keys specified.',
     )
     api_version: Optional[str] = Field(
-        description="If specified, overrides default version used by the Salesforce package. Example value: '59.0'"
+        None,
+        description="If specified, overrides default version used by the Salesforce package. Example value: '59.0'",
     )
 
     profiling: SalesforceProfilingConfig = SalesforceProfilingConfig()
@@ -520,7 +527,7 @@ class SalesforceApi:
 
 @platform_name("Salesforce")
 @config_class(SalesforceConfig)
-@support_status(SupportStatus.INCUBATING)
+@support_status(SupportStatus.CERTIFIED)
 @capability(
     capability_name=SourceCapability.PLATFORM_INSTANCE,
     description="Can be equivalent to Salesforce organization",
@@ -532,11 +539,11 @@ class SalesforceApi:
 @capability(
     capability_name=SourceCapability.DATA_PROFILING,
     description="Only table level profiling is supported via `profiling.enabled` config field",
+    subtype_modifier=[SourceCapabilityModifier.TABLE],
 )
 @capability(
     capability_name=SourceCapability.DELETION_DETECTION,
-    description="Not supported yet",
-    supported=False,
+    description="Enabled by default via stateful ingestion",
 )
 @capability(
     capability_name=SourceCapability.SCHEMA_METADATA,
@@ -546,6 +553,14 @@ class SalesforceApi:
     capability_name=SourceCapability.TAGS,
     description="Enabled by default",
 )
+@capability(
+    capability_name=SourceCapability.LINEAGE_COARSE,
+    description="Extract table-level lineage for Salesforce objects",
+    subtype_modifier=[
+        SourceCapabilityModifier.SALESFORCE_CUSTOM_OBJECT,
+        SourceCapabilityModifier.SALESFORCE_STANDARD_OBJECT,
+    ],
+)
 class SalesforceSource(StatefulIngestionSourceBase):
     def __init__(self, config: SalesforceConfig, ctx: PipelineContext) -> None:
         super().__init__(config, ctx)

datahub/ingestion/source/schema/json_schema.py

@@ -4,7 +4,6 @@ import logging
 import os
 import tempfile
 import unittest
-import urllib.request
 from dataclasses import dataclass
 from os.path import basename, dirname
 from pathlib import Path
@@ -12,6 +11,7 @@ from typing import Any, Iterable, List, Optional, Union
 from urllib.parse import urlparse
 
 import jsonref
+import requests
 from pydantic import AnyHttpUrl, DirectoryPath, FilePath, validator
 from pydantic.fields import Field
 
@@ -91,19 +91,18 @@ class JsonSchemaSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMix
     )
 
     @validator("path")
-    def download_http_url_to_temp_file(v):
+    def download_http_url_to_temp_file(cls, v):
         if isinstance(v, AnyHttpUrl):
             try:
-                with urllib.request.urlopen(v) as response:
-                    schema_dict = json.load(response)
-                    if not JsonSchemaTranslator._get_id_from_any_schema(schema_dict):
-                        schema_dict["$id"] = str(v)
-                    with tempfile.NamedTemporaryFile(
-                        mode="w", delete=False
-                    ) as tmp_file:
-                        tmp_file.write(json.dumps(schema_dict))
-                        tmp_file.flush()
-                        return tmp_file.name
+                response = requests.get(str(v))
+                response.raise_for_status()
+                schema_dict = response.json()
+                if not JsonSchemaTranslator._get_id_from_any_schema(schema_dict):
+                    schema_dict["$id"] = str(v)
+                with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
+                    tmp_file.write(json.dumps(schema_dict))
+                    tmp_file.flush()
+                    return tmp_file.name
             except Exception as e:
                 logger.error(
                     f"Failed to localize url {v} due to {e}. Run with --debug to get full stacktrace"
@@ -353,7 +352,7 @@ class JsonSchemaSource(StatefulIngestionSourceBase):
         if self.config.platform_instance:
             browse_prefix = f"/{self.config.env.lower()}/{self.config.platform}/{self.config.platform_instance}"
 
-        if os.path.isdir(self.config.path):
+        if isinstance(self.config.path, Path) and self.config.path.is_dir():
             for root, _, files in os.walk(self.config.path, topdown=False):
                 for file_name in [f for f in files if f.endswith(".json")]:
                     try:
@@ -373,10 +372,11 @@ class JsonSchemaSource(StatefulIngestionSourceBase):
 
         else:
             try:
+                assert isinstance(self.config.path, Path)
                 yield from self._load_one_file(
                     ref_loader,
                     browse_prefix=browse_prefix,
-                    root_dir=Path(os.path.dirname(Path(self.config.path))),
+                    root_dir=self.config.path.parent,
                     file_name=str(self.config.path),
                 )
             except Exception as e:

datahub/ingestion/source/schema_inference/object.py

@@ -1,4 +1,4 @@
-from collections import Counter
+from collections import Counter, defaultdict
 from typing import Any, Counter as CounterType, Dict, Sequence, Tuple, Union
 
 from typing_extensions import TypedDict
@@ -84,7 +84,7 @@ def is_nullable_collection(
 
 
 def construct_schema(
-    collection: Sequence[Dict[str, Any]], delimiter: str
+    collection: Sequence[Dict[str, Any]], delimiter: str = "."
 ) -> Dict[Tuple[str, ...], SchemaDescription]:
     """
     Construct (infer) a schema from a collection of documents.
@@ -104,9 +104,11 @@ def construct_schema(
             string to concatenate field names by
     """
 
-    schema: Dict[Tuple[str, ...], BasicSchemaDescription] = {}
+    schema: Dict[Tuple[str, ...], BasicSchemaDescription] = defaultdict(
+        lambda: {"types": Counter(), "count": 0}
+    )
 
-    def append_to_schema(doc: Dict[str, Any], parent_prefix: Tuple[str, ...]) -> None:
+    def append_to_schema(doc: Dict[str, Any], parent_prefix: Tuple[str, ...]) -> int:
         """
         Recursively update the schema with a document, which may/may not contain nested fields.
 
@@ -118,18 +120,24 @@ def construct_schema(
                 prefix of fields that the document is under, pass an empty tuple when initializing
         """
 
+        # we want to make sure that parents of nested structures are included first, before their children, so that
+        # they are displayed properly in the UI, also in the event of trimming the list (which happens, for example,
+        # in mongodb ingestor)
+        max_count = 0
         for key, value in doc.items():
             new_parent_prefix = parent_prefix + (key,)
 
             # if nested value, look at the types within
             if isinstance(value, dict):
-                append_to_schema(value, new_parent_prefix)
+                max_count = max(append_to_schema(value, new_parent_prefix), max_count)
             # if array of values, check what types are within
             if isinstance(value, list):
                 for item in value:
                     # if dictionary, add it as a nested object
                     if isinstance(item, dict):
-                        append_to_schema(item, new_parent_prefix)
+                        max_count = max(
+                            append_to_schema(item, new_parent_prefix), max_count
+                        )
 
             # don't record None values (counted towards nullable)
             if value is not None:
@@ -143,6 +151,14 @@ def construct_schema(
                     # update the type count
                     schema[new_parent_prefix]["types"].update({type(value): 1})
                     schema[new_parent_prefix]["count"] += 1
+            max_count = max(schema[new_parent_prefix]["count"], max_count)
+
+        if parent_prefix != ():
+            schema[parent_prefix]["count"] = max(
+                schema[parent_prefix]["count"], max_count
+            )
+
+        return max_count
 
     for document in collection:
         append_to_schema(document, ())

datahub/ingestion/source/sigma/config.py

@@ -1,14 +1,16 @@
 import logging
 from dataclasses import dataclass, field
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 import pydantic
+from pydantic import BaseModel, Field
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.source_common import (
     EnvConfigMixin,
     PlatformInstanceConfigMixin,
 )
+from datahub.ingestion.api.report import EntityFilterReport
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalSourceReport,
     StatefulStaleMetadataRemovalConfig,
@@ -52,17 +54,82 @@ class Constant:
     DEFAULT_API_URL = "https://aws-api.sigmacomputing.com/v2"
 
 
+class WorkspaceCounts(BaseModel):
+    workbooks_count: int = 0
+    datasets_count: int = 0
+    elements_count: int = 0
+    pages_count: int = 0
+
+    def is_empty(self) -> bool:
+        return (
+            self.workbooks_count == 0
+            and self.datasets_count == 0
+            and self.elements_count == 0
+            and self.pages_count == 0
+        )
+
+    def as_obj(self) -> dict:
+        return {
+            "workbooks_count": self.workbooks_count,
+            "datasets_count": self.datasets_count,
+            "elements_count": self.elements_count,
+            "pages_count": self.pages_count,
+        }
+
+
+class SigmaWorkspaceEntityFilterReport(EntityFilterReport):
+    type: str = "workspace"
+
+    workspace_counts: Dict[str, WorkspaceCounts] = Field(
+        default_factory=dict,
+        description="Counts of workbooks, datasets, elements and pages in each workspace.",
+    )
+
+    def increment_workbooks_count(self, workspace_id: str) -> None:
+        if workspace_id not in self.workspace_counts:
+            self.workspace_counts[workspace_id] = WorkspaceCounts()
+        self.workspace_counts[workspace_id].workbooks_count += 1
+
+    def increment_datasets_count(self, workspace_id: str) -> None:
+        if workspace_id not in self.workspace_counts:
+            self.workspace_counts[workspace_id] = WorkspaceCounts()
+        self.workspace_counts[workspace_id].datasets_count += 1
+
+    def increment_elements_count(self, workspace_id: str) -> None:
+        if workspace_id not in self.workspace_counts:
+            self.workspace_counts[workspace_id] = WorkspaceCounts()
+        self.workspace_counts[workspace_id].elements_count += 1
+
+    def increment_pages_count(self, workspace_id: str) -> None:
+        if workspace_id not in self.workspace_counts:
+            self.workspace_counts[workspace_id] = WorkspaceCounts()
+        self.workspace_counts[workspace_id].pages_count += 1
+
+    def as_obj(self) -> dict:
+        return {
+            "filtered": self.dropped_entities.as_obj(),
+            "processed": self.processed_entities.as_obj(),
+            "workspace_counts": {
+                key: item.as_obj() for key, item in self.workspace_counts.items()
+            },
+        }
+
+
 @dataclass
 class SigmaSourceReport(StaleEntityRemovalSourceReport):
-    number_of_workspaces: int = 0
+    workspaces: SigmaWorkspaceEntityFilterReport = field(
+        default_factory=SigmaWorkspaceEntityFilterReport
+    )
     non_accessible_workspaces_count: int = 0
-    shared_entities_count: int = 0
-    number_of_datasets: int = 0
-    number_of_workbooks: int = 0
-    number_of_files_metadata: Dict[str, int] = field(default_factory=dict)
 
-    def report_number_of_workspaces(self, number_of_workspaces: int) -> None:
-        self.number_of_workspaces = number_of_workspaces
+    datasets: EntityFilterReport = EntityFilterReport.field(type="dataset")
+    datasets_without_workspace: int = 0
+
+    workbooks: EntityFilterReport = EntityFilterReport.field(type="workbook")
+    workbooks_without_workspace: int = 0
+
+    number_of_files_metadata: Dict[str, int] = field(default_factory=dict)
+    empty_workspaces: List[str] = field(default_factory=list)
 
 
 class PlatformDetail(PlatformInstanceConfigMixin, EnvConfigMixin):

datahub/ingestion/source/sigma/data_classes.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from datetime import datetime
 from typing import Dict, List, Optional
 
@@ -23,6 +24,8 @@ class Workspace(BaseModel):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
         # Update name if presonal workspace
         if values["name"] == "User Folder":
             values["name"] = "My documents"

datahub/ingestion/source/sigma/sigma.py

@@ -30,11 +30,13 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import (
     BIContainerSubTypes,
     DatasetSubTypes,
+    SourceCapabilityModifier,
 )
 from datahub.ingestion.source.sigma.config import (
     PlatformDetail,
     SigmaSourceConfig,
     SigmaSourceReport,
+    WorkspaceCounts,
 )
 from datahub.ingestion.source.sigma.data_classes import (
     Element,
@@ -94,7 +96,11 @@ logger = logging.getLogger(__name__)
 @platform_name("Sigma")
 @config_class(SigmaSourceConfig)
 @support_status(SupportStatus.INCUBATING)
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+    SourceCapability.CONTAINERS,
+    "Enabled by default",
+    subtype_modifier=[SourceCapabilityModifier.SIGMA_WORKSPACE],
+)
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default.")
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
@@ -104,6 +110,7 @@ logger = logging.getLogger(__name__)
     SourceCapability.OWNERSHIP,
     "Enabled by default, configured using `ingest_owner`",
 )
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class SigmaSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin extracts the following:
@@ -162,14 +169,18 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
 
     def _get_allowed_workspaces(self) -> List[Workspace]:
         all_workspaces = self.sigma_api.workspaces.values()
-        allowed_workspaces = [
-            workspace
-            for workspace in all_workspaces
-            if self.config.workspace_pattern.allowed(workspace.name)
-        ]
         logger.info(f"Number of workspaces = {len(all_workspaces)}")
-        self.reporter.report_number_of_workspaces(len(all_workspaces))
+
+        allowed_workspaces = []
+        for workspace in all_workspaces:
+            if self.config.workspace_pattern.allowed(workspace.name):
+                allowed_workspaces.append(workspace)
+            else:
+                self.reporter.workspaces.dropped(
+                    f"{workspace.name} ({workspace.workspaceId})"
+                )
         logger.info(f"Number of allowed workspaces = {len(allowed_workspaces)}")
+
         return allowed_workspaces
 
     def _gen_workspace_workunit(
@@ -280,6 +291,7 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
         yield self._gen_dataset_properties(dataset_urn, dataset)
 
         if dataset.workspaceId:
+            self.reporter.workspaces.increment_datasets_count(dataset.workspaceId)
             yield from add_entity_to_container(
                 container_key=self._gen_workspace_key(dataset.workspaceId),
                 entity_type="dataset",
@@ -463,6 +475,8 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
             ).as_workunit()
 
             if workbook.workspaceId:
+                self.reporter.workspaces.increment_elements_count(workbook.workspaceId)
+
                 yield self._gen_entity_browsepath_aspect(
                     entity_urn=chart_urn,
                     parent_entity_urn=builder.make_container_urn(
@@ -520,6 +534,7 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
             all_input_fields: List[InputFieldClass] = []
 
             if workbook.workspaceId:
+                self.reporter.workspaces.increment_pages_count(workbook.workspaceId)
                 yield self._gen_entity_browsepath_aspect(
                     entity_urn=dashboard_urn,
                     parent_entity_urn=builder.make_container_urn(
@@ -609,6 +624,8 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
 
         paths = workbook.path.split("/")[1:]
         if workbook.workspaceId:
+            self.reporter.workspaces.increment_workbooks_count(workbook.workspaceId)
+
             yield self._gen_entity_browsepath_aspect(
                 entity_urn=dashboard_urn,
                 parent_entity_urn=builder.make_container_urn(
@@ -658,7 +675,19 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
             yield from self._gen_workbook_workunit(workbook)
 
         for workspace in self._get_allowed_workspaces():
+            self.reporter.workspaces.processed(
+                f"{workspace.name} ({workspace.workspaceId})"
+            )
             yield from self._gen_workspace_workunit(workspace)
+            if self.reporter.workspaces.workspace_counts.get(
+                workspace.workspaceId, WorkspaceCounts()
+            ).is_empty():
+                logger.warning(
+                    f"Workspace {workspace.name} ({workspace.workspaceId}) is empty. If this is not expected, add the user associated with the Client ID/Secret to each workspace with missing metadata"
+                )
+                self.reporter.empty_workspaces.append(
+                    f"{workspace.name} ({workspace.workspaceId})"
+                )
         yield from self._gen_sigma_dataset_upstream_lineage_workunit()
 
     def get_report(self) -> SourceReport: