acryl-datahub 1.2.0.9rc1__py3-none-any.whl → 1.2.0.10__py3-none-any.whl

datahub/ingestion/source/redshift/query.py

@@ -89,7 +89,7 @@ class RedshiftCommonQuery:
     ) -> str:
         # NOTE: it looks like description is available only in pg_description
         # So this remains preferrred way
-        tables_query = """
+        tables_query = f"""
  SELECT  CASE c.relkind
                 WHEN 'r' THEN 'TABLE'
                 WHEN 'v' THEN 'VIEW'
@@ -120,6 +120,7 @@ class RedshiftCommonQuery:
         LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
         LEFT JOIN pg_class_info as ci on c.oid = ci.reloid
         LEFT JOIN pg_catalog.pg_description pgd ON pgd.objsubid = 0 AND pgd.objoid = c.oid
+        JOIN svv_redshift_schemas rs ON rs.schema_name = n.nspname AND rs.database_name = '{database}'
         WHERE c.relkind IN ('r','v','m','S','f')
         AND   n.nspname !~ '^pg_'
         AND   n.nspname != 'information_schema'
@@ -128,23 +129,24 @@ class RedshiftCommonQuery:
         external_tables_query = f"""
         SELECT 'EXTERNAL_TABLE' as tabletype,
             NULL AS "schema_oid",
-            schemaname AS "schema",
+            t.schemaname AS "schema",
             NULL AS "rel_oid",
-            tablename AS "relname",
+            t.tablename AS "relname",
             NULL as "creation_time",
             NULL AS "diststyle",
             NULL AS "owner_id",
             NULL AS "owner_name",
             NULL AS "view_definition",
             NULL AS "privileges",
-            "location",
-            parameters,
-            input_format,
-            output_format,
-            serde_parameters,
+            t."location",
+            t.parameters,
+            t.input_format,
+            t.output_format,
+            t.serde_parameters,
             NULL as table_description
-        FROM pg_catalog.svv_external_tables
-        WHERE redshift_database_name='{database}'
+        FROM pg_catalog.svv_external_tables t
+        JOIN SVV_EXTERNAL_SCHEMAS s ON t.schemaname = s.schemaname
+        WHERE t.redshift_database_name='{database}'
         ORDER BY "schema",
                 "relname"
 """
@@ -232,11 +234,12 @@ class RedshiftCommonQuery:
               ON att.attrelid = c.oid
             LEFT JOIN pg_catalog.pg_attrdef ad
               ON (att.attrelid, att.attnum) = (ad.adrelid, ad.adnum)
+            JOIN svv_redshift_schemas rs ON rs.schema_name = n.nspname AND rs.database_name = '{database_name}'
             WHERE n.nspname !~ '^pg_'
               AND   n.nspname != 'information_schema'
               AND att.attnum > 0
               AND NOT att.attisdropped
-              and schema = '{schema_name}'
+              and n.nspname = '{schema_name}'
             UNION
             SELECT
               view_schema as "schema",
@@ -263,26 +266,27 @@ class RedshiftCommonQuery:
             WHERE 1 and schema = '{schema_name}'
             UNION
             SELECT
-              schemaname as "schema",
-              tablename as "table_name",
-              columnname as "name",
+              c.schemaname as "schema",
+              c.tablename as "table_name",
+              c.columnname as "name",
               null as "encode",
               -- Spectrum represents data types differently.
               -- Standardize, so we can infer types.
-              external_type AS "type",
+              c.external_type AS "type",
               null as "distkey",
               0 as "sortkey",
               null as "notnull",
               null as "comment",
               null as "adsrc",
               null as "attnum",
-              external_type AS "format_type",
+              c.external_type AS "format_type",
               null as "default",
               null as "schema_oid",
               null as "table_oid"
-            FROM SVV_EXTERNAL_COLUMNS
-            WHERE 1 and schema = '{schema_name}'
-            AND redshift_database_name = '{database_name}'
+            FROM SVV_EXTERNAL_COLUMNS c
+            JOIN SVV_EXTERNAL_SCHEMAS s ON c.schemaname = s.schemaname
+            WHERE c.schemaname = '{schema_name}'
+            AND c.redshift_database_name = '{database_name}'
             ORDER BY "schema", "table_name", "attnum"
 """
 

datahub/ingestion/source/s3/source.py

@@ -115,14 +115,7 @@ profiling_flags_to_report = [
     "include_field_sample_values",
 ]
 
-
-# LOCAL_BROWSE_PATH_TRANSFORMER_CONFIG = AddDatasetBrowsePathConfig(
-#     path_templates=["/ENV/PLATFORMDATASET_PARTS"], replace_existing=True
-# )
-#
-# LOCAL_BROWSE_PATH_TRANSFORMER = AddDatasetBrowsePathTransformer(
-#     ctx=None, config=LOCAL_BROWSE_PATH_TRANSFORMER_CONFIG
-# )
+URI_SCHEME_REGEX = re.compile(r"^[a-z0-9]+://")
 
 
 def partitioned_folder_comparator(folder1: str, folder2: str) -> int:
@@ -448,9 +441,8 @@ class S3Source(StatefulIngestionSourceBase):
                 self.source_config.verify_ssl
             )
 
-            file = smart_open(
-                table_data.full_path, "rb", transport_params={"client": s3_client}
-            )
+            path = re.sub(URI_SCHEME_REGEX, "s3://", table_data.full_path)
+            file = smart_open(path, "rb", transport_params={"client": s3_client})
         else:
             # We still use smart_open here to take advantage of the compression
             # capabilities of smart_open.
@@ -668,11 +660,9 @@ class S3Source(StatefulIngestionSourceBase):
         aspects: List[Optional[_Aspect]] = []
 
         logger.info(f"Extracting table schema from file: {table_data.full_path}")
-        browse_path: str = (
-            self.strip_s3_prefix(table_data.table_path)
-            if self.is_s3_platform()
-            else table_data.table_path.strip("/")
-        )
+
+        # remove protocol and any leading or trailing slashes
+        browse_path = re.sub(URI_SCHEME_REGEX, "", table_data.table_path).strip("/")
 
         data_platform_urn = make_data_platform_urn(self.source_config.platform)
         logger.info(f"Creating dataset urn with name: {browse_path}")
@@ -806,10 +796,20 @@ class S3Source(StatefulIngestionSourceBase):
         else:
             return relative_path
 
-    def extract_table_name(self, path_spec: PathSpec, named_vars: dict) -> str:
-        if path_spec.table_name is None:
-            raise ValueError("path_spec.table_name is not set")
-        return path_spec.table_name.format_map(named_vars)
+    def extract_table_name_and_path(
+        self, path_spec: PathSpec, path: str
+    ) -> Tuple[str, str]:
+        # Extract the table name and base path from a path that's been normalized back to the
+        # "s3://" scheme that matches the path_spec
+        table_name, table_path = path_spec.extract_table_name_and_path(
+            self._normalize_uri_for_pattern_matching(path)
+        )
+        # Then convert the table base path back to the original scheme
+        scheme = re.match(URI_SCHEME_REGEX, path)
+        if scheme:
+            table_path = re.sub(URI_SCHEME_REGEX, scheme[0], table_path)
+
+        return table_name, table_path
 
     def extract_table_data(
         self,
@@ -819,7 +819,7 @@ class S3Source(StatefulIngestionSourceBase):
         path = browse_path.file
         partitions = browse_path.partitions
         logger.debug(f"Getting table data for path: {path}")
-        table_name, table_path = path_spec.extract_table_name_and_path(path)
+        table_name, table_path = self.extract_table_name_and_path(path_spec, path)
         return TableData(
             display_name=table_name,
             is_s3=self.is_s3_platform(),
@@ -992,7 +992,9 @@ class S3Source(StatefulIngestionSourceBase):
             )
 
             # If partition_id is None, it means the folder is not a partition
-            partition_id = path_spec.get_partition_from_path(max_file_s3_path)
+            partition_id = path_spec.get_partition_from_path(
+                self._normalize_uri_for_pattern_matching(max_file_s3_path)
+            )
 
             yield Folder(
                 partition_id=partition_id,
@@ -1143,8 +1145,8 @@ class S3Source(StatefulIngestionSourceBase):
 
                     # Extract table name using the ORIGINAL path spec pattern matching (not the modified one)
                     # This uses the compiled regex pattern to extract the table name from the full path
-                    table_name, table_path = path_spec.extract_table_name_and_path(
-                        table_s3_path
+                    table_name, _ = self.extract_table_name_and_path(
+                        path_spec, table_s3_path
                     )
 
                     # Apply table name filtering if configured

datahub/ingestion/source/salesforce.py

@@ -110,30 +110,33 @@ class SalesforceConfig(
     auth: SalesforceAuthType = SalesforceAuthType.USERNAME_PASSWORD
 
     # Username, Password Auth
-    username: Optional[str] = Field(description="Salesforce username")
-    password: Optional[str] = Field(description="Password for Salesforce user")
+    username: Optional[str] = Field(None, description="Salesforce username")
+    password: Optional[str] = Field(None, description="Password for Salesforce user")
     consumer_key: Optional[str] = Field(
-        description="Consumer key for Salesforce JSON web token access"
+        None, description="Consumer key for Salesforce JSON web token access"
     )
     private_key: Optional[str] = Field(
-        description="Private key as a string for Salesforce JSON web token access"
+        None, description="Private key as a string for Salesforce JSON web token access"
     )
     security_token: Optional[str] = Field(
-        description="Security token for Salesforce username"
+        None, description="Security token for Salesforce username"
     )
     # client_id, client_secret not required
 
     # Direct - Instance URL, Access Token Auth
     instance_url: Optional[str] = Field(
-        description="Salesforce instance url. e.g. https://MyDomainName.my.salesforce.com"
+        None,
+        description="Salesforce instance url. e.g. https://MyDomainName.my.salesforce.com",
     )
     # Flag to indicate whether the instance is production or sandbox
     is_sandbox: bool = Field(
         default=False, description="Connect to Sandbox instance of your Salesforce"
     )
-    access_token: Optional[str] = Field(description="Access token for instance url")
+    access_token: Optional[str] = Field(
+        None, description="Access token for instance url"
+    )
 
-    ingest_tags: Optional[bool] = Field(
+    ingest_tags: bool = Field(
         default=False,
         description="Ingest Tags from source. This will override Tags entered from UI",
     )
@@ -147,7 +150,8 @@ class SalesforceConfig(
         description='Regex patterns for tables/schemas to describe domain_key domain key (domain_key can be any string like "sales".) There can be multiple domain keys specified.',
     )
     api_version: Optional[str] = Field(
-        description="If specified, overrides default version used by the Salesforce package. Example value: '59.0'"
+        None,
+        description="If specified, overrides default version used by the Salesforce package. Example value: '59.0'",
     )
 
     profiling: SalesforceProfilingConfig = SalesforceProfilingConfig()

datahub/ingestion/source/schema/json_schema.py

@@ -4,7 +4,6 @@ import logging
 import os
 import tempfile
 import unittest
-import urllib.request
 from dataclasses import dataclass
 from os.path import basename, dirname
 from pathlib import Path
@@ -12,6 +11,7 @@ from typing import Any, Iterable, List, Optional, Union
 from urllib.parse import urlparse
 
 import jsonref
+import requests
 from pydantic import AnyHttpUrl, DirectoryPath, FilePath, validator
 from pydantic.fields import Field
 
@@ -91,19 +91,18 @@ class JsonSchemaSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMix
     )
 
     @validator("path")
-    def download_http_url_to_temp_file(v):
+    def download_http_url_to_temp_file(cls, v):
         if isinstance(v, AnyHttpUrl):
             try:
-                with urllib.request.urlopen(v) as response:
-                    schema_dict = json.load(response)
-                    if not JsonSchemaTranslator._get_id_from_any_schema(schema_dict):
-                        schema_dict["$id"] = str(v)
-                    with tempfile.NamedTemporaryFile(
-                        mode="w", delete=False
-                    ) as tmp_file:
-                        tmp_file.write(json.dumps(schema_dict))
-                        tmp_file.flush()
-                        return tmp_file.name
+                response = requests.get(str(v))
+                response.raise_for_status()
+                schema_dict = response.json()
+                if not JsonSchemaTranslator._get_id_from_any_schema(schema_dict):
+                    schema_dict["$id"] = str(v)
+                with tempfile.NamedTemporaryFile(mode="w", delete=False) as tmp_file:
+                    tmp_file.write(json.dumps(schema_dict))
+                    tmp_file.flush()
+                    return tmp_file.name
             except Exception as e:
                 logger.error(
                     f"Failed to localize url {v} due to {e}. Run with --debug to get full stacktrace"
@@ -353,7 +352,7 @@ class JsonSchemaSource(StatefulIngestionSourceBase):
         if self.config.platform_instance:
             browse_prefix = f"/{self.config.env.lower()}/{self.config.platform}/{self.config.platform_instance}"
 
-        if os.path.isdir(self.config.path):
+        if isinstance(self.config.path, Path) and self.config.path.is_dir():
             for root, _, files in os.walk(self.config.path, topdown=False):
                 for file_name in [f for f in files if f.endswith(".json")]:
                     try:
@@ -373,10 +372,11 @@ class JsonSchemaSource(StatefulIngestionSourceBase):
 
         else:
             try:
+                assert isinstance(self.config.path, Path)
                 yield from self._load_one_file(
                     ref_loader,
                     browse_prefix=browse_prefix,
-                    root_dir=Path(os.path.dirname(Path(self.config.path))),
+                    root_dir=self.config.path.parent,
                     file_name=str(self.config.path),
                 )
             except Exception as e:

datahub/ingestion/source/sigma/data_classes.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from datetime import datetime
 from typing import Dict, List, Optional
 
@@ -23,6 +24,8 @@ class Workspace(BaseModel):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
         # Update name if presonal workspace
         if values["name"] == "User Folder":
             values["name"] = "My documents"