castor-extractor 0.17.4__py3-none-any.whl → 0.18.2__py3-none-any.whl

castor_extractor/visualization/qlik/extract.py

@@ -7,13 +7,11 @@ from ...utils import (
     deep_serialize,
     from_env,
     get_output_filename,
-    validate_baseurl,
     write_json,
     write_summary,
 )
 from .assets import QlikAsset
-from .client import QlikClient
-from .constants import API_KEY, BASE_URL
+from .client import QlikClient, QlikCredentials
 
 logger = logging.getLogger(__name__)
 
@@ -59,13 +57,11 @@ def extract_all(
     Store the output files locally under the given output_directory
     """
 
+    credentials = QlikCredentials(base_url=base_url, api_key=api_key)
     _output_directory = output_directory or from_env(OUTPUT_DIR)
-    _base_url = validate_baseurl(base_url or from_env(BASE_URL))
-    _api_key = api_key or from_env(API_KEY)
 
     client = QlikClient(
-        server_url=_base_url,
-        api_key=_api_key,
+        credentials=credentials,
         except_http_error_statuses=except_http_error_statuses,
     )
 
@@ -75,4 +71,4 @@ def extract_all(
         filename = get_output_filename(key.name.lower(), _output_directory, ts)
         write_json(filename, data)
 
-    write_summary(_output_directory, ts, base_url=_base_url)
+    write_summary(_output_directory, ts, base_url=credentials.base_url)

castor_extractor/visualization/sigma/__init__.py

@@ -1,3 +1,3 @@
 from .assets import SigmaAsset
-from .client import CredentialsKey, SigmaClient, SigmaCredentials
+from .client import SigmaClient, SigmaCredentials
 from .extract import extract_all

castor_extractor/visualization/sigma/client/__init__.py

@@ -1,2 +1,2 @@
 from .client import SigmaClient
-from .credentials import CredentialsKey, SigmaCredentials
+from .credentials import SigmaCredentials

castor_extractor/visualization/sigma/client/client.py

@@ -5,7 +5,7 @@ from urllib.parse import urljoin
 import requests
 
 from ..assets import SigmaAsset
-from .credentials import CredentialsKey, SigmaCredentials
+from .credentials import SigmaCredentials
 from .endpoints import EndpointFactory
 from .pagination import Pagination
 
@@ -29,6 +29,7 @@ class SigmaClient:
         self.host = credentials.host
         self.client_id = credentials.client_id
         self.api_token = credentials.api_token
+        self.grant_type = credentials.grant_type
         self.headers: Optional[Dict[str, str]] = None
 
     def _get_token(self) -> Dict[str, str]:
@@ -37,9 +38,9 @@ class SigmaClient:
         token_response = requests.post(  # noqa: S113
             token_api_path,
             data={
-                CredentialsKey.GRANT_TYPE.value: "client_credentials",
-                CredentialsKey.CLIENT_ID.value: self.client_id,
-                CredentialsKey.CLIENT_SECRET.value: self.api_token,
+                "grant_type": self.grant_type,
+                "client_id": self.client_id,
+                "client_secret": self.api_token,
             },
         )
         if token_response.status_code != requests.codes.OK:

castor_extractor/visualization/sigma/client/credentials.py

@@ -1,35 +1,19 @@
-from dataclasses import dataclass
-from enum import Enum
+from pydantic import Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
+CASTOR_ENV_PREFIX = "CASTOR_SIGMA_"
 
-class CredentialsKey(Enum):
-    """Value enum object for the credentials"""
 
-    CLIENT_SECRET = "client_secret"  # noqa: S105
-    CLIENT_ID = "client_id"
-    HOST = "host"
-    GRANT_TYPE = "grant_type"
-    API_TOKEN = "api_token"  # noqa: S105
-
-
-CLIENT_ALLOWED_KEYS = (
-    CredentialsKey.HOST.value,
-    CredentialsKey.CLIENT_ID.value,
-    CredentialsKey.API_TOKEN.value,
-)
-
-
-@dataclass
-class SigmaCredentials:
+class SigmaCredentials(BaseSettings):
     """Class to handle Sigma rest API permissions"""
 
-    api_token: str
+    model_config = SettingsConfigDict(
+        env_prefix=CASTOR_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    api_token: str = Field(repr=False)
     client_id: str
     host: str
-
-    @classmethod
-    def from_secret(cls, secret: dict) -> "SigmaCredentials":
-        credentials = {
-            k: v for k, v in secret.items() if k in CLIENT_ALLOWED_KEYS
-        }
-        return cls(**credentials)
+    grant_type: str = "client_credentials"

castor_extractor/visualization/sigma/extract.py

@@ -12,7 +12,6 @@ from ...utils import (
 )
 from .assets import SigmaAsset
 from .client import SigmaClient, SigmaCredentials
-from .constants import API_TOKEN, CLIENT_ID, HOST
 
 logger = logging.getLogger(__name__)
 
@@ -63,14 +62,11 @@ def extract_all(
     """
 
     _output_directory = output_directory or from_env(OUTPUT_DIR)
-    _client_id = client_id or from_env(CLIENT_ID)
-    _host = host or from_env(HOST)
-    _api_token = api_token or from_env(API_TOKEN)
 
     credentials = SigmaCredentials(
-        host=_host,
-        client_id=_client_id,
-        api_token=_api_token,
+        host=host,
+        client_id=client_id,
+        api_token=api_token,
     )
     client = SigmaClient(credentials=credentials)
 
@@ -80,4 +76,4 @@ def extract_all(
         filename = get_output_filename(key.name.lower(), _output_directory, ts)
         write_json(filename, data)
 
-    write_summary(_output_directory, ts, host=_host)
+    write_summary(_output_directory, ts, host=credentials.host)

castor_extractor/visualization/tableau_revamp/client/credentials.py

@@ -1,104 +1,57 @@
-from enum import Enum
-from typing import Dict, Literal, Optional, overload
+from typing import Optional
 
-from ....utils import from_env
-
-_AUTH_ERROR_MSG = "Need either user and password or token_name and token"
+from pydantic import field_validator, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 # To specify the default site on Tableau Server, you can use an empty string
 # https://tableau.github.io/server-client-python/docs/api-ref#authentication
 _DEFAULT_SERVER_SITE_ID = ""
 
-
-class CredentialsKey(Enum):
-    """Value enum object for the credentials"""
-
-    TABLEAU_USER = "user"
-    TABLEAU_PASSWORD = "password"  # noqa: S105
-    TABLEAU_TOKEN_NAME = "token_name"  # noqa: S105
-    TABLEAU_TOKEN = "token"  # noqa: S105
-    TABLEAU_SITE_ID = "site_id"
-    TABLEAU_SERVER_URL = "server_url"
-
-
-CREDENTIALS_ENV: Dict[CredentialsKey, str] = {
-    CredentialsKey.TABLEAU_USER: "CASTOR_TABLEAU_USER",
-    CredentialsKey.TABLEAU_PASSWORD: "CASTOR_TABLEAU_PASSWORD",
-    CredentialsKey.TABLEAU_TOKEN_NAME: "CASTOR_TABLEAU_TOKEN_NAME",
-    CredentialsKey.TABLEAU_TOKEN: "CASTOR_TABLEAU_TOKEN",
-    CredentialsKey.TABLEAU_SITE_ID: "CASTOR_TABLEAU_SITE_ID",
-    CredentialsKey.TABLEAU_SERVER_URL: "CASTOR_TABLEAU_SERVER_URL",
-}
-
-
-@overload
-def get_value(key: CredentialsKey, kwargs: dict) -> Optional[str]: ...
+# In Castor APP, site_id is mandatory: users can't let this field empty
+# In that case, we encourage users to write "Default" instead
+_DEFAULT_SITE_ID_USER_INPUT = "default"
 
 
-@overload
-def get_value(
-    key: CredentialsKey, kwargs: dict, optional: Literal[True]
-) -> Optional[str]: ...
+TABLEAU_ENV_PREFIX = "CASTOR_TABLEAU_"
 
 
-@overload
-def get_value(
-    key: CredentialsKey, kwargs: dict, optional: Literal[False]
-) -> str: ...
-
-
-def get_value(
-    key: CredentialsKey,
-    kwargs: dict,
-    optional: bool = True,
-) -> Optional[str]:
+class TableauRevampCredentials(BaseSettings):
     """
-    Returns the value of the given key:
-    - from kwargs in priority
-    - from ENV otherwise
-    Raises an error if not found (unless optional)
+    Tableau's credentials to connect to both APIs (REST and GRAPHQL)
     """
 
-    if key.value in kwargs:
-        return kwargs[key.value]
-
-    env_key = CREDENTIALS_ENV[key]
-    return from_env(env_key, optional)
+    model_config = SettingsConfigDict(
+        env_prefix=TABLEAU_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
 
+    server_url: str
+    site_id: str = ""
 
-class TableauRevampCredentials:
-    """
-    Tableau's credentials to connect to REST API
-    """
-
-    def __init__(
-        self,
-        *,
-        server_url: str,
-        site_id: Optional[str],
-        user: Optional[str],
-        password: Optional[str],
-        token_name: Optional[str],
-        token: Optional[str],
-    ):
-        self.user = user
-        self.site_id = site_id or _DEFAULT_SERVER_SITE_ID
-        self.server_url = server_url
-        self.password = password
-        self.token_name = token_name
-        self.token = token
+    password: Optional[str] = None
+    token: Optional[str] = None
+    token_name: Optional[str] = None
+    user: Optional[str] = None
 
+    @field_validator("site_id", mode="before")
     @classmethod
-    def from_env(cls, kwargs: dict) -> "TableauRevampCredentials":
-        return TableauRevampCredentials(
-            server_url=get_value(
-                CredentialsKey.TABLEAU_SERVER_URL,
-                kwargs,
-                optional=False,
-            ),
-            site_id=get_value(CredentialsKey.TABLEAU_SITE_ID, kwargs),
-            user=get_value(CredentialsKey.TABLEAU_USER, kwargs),
-            password=get_value(CredentialsKey.TABLEAU_PASSWORD, kwargs),
-            token_name=get_value(CredentialsKey.TABLEAU_TOKEN_NAME, kwargs),
-            token=get_value(CredentialsKey.TABLEAU_TOKEN, kwargs),
-        )
+    def _check_site_id(cls, site_id: Optional[str]) -> str:
+        if not site_id or site_id.lower() == _DEFAULT_SITE_ID_USER_INPUT:
+            return _DEFAULT_SERVER_SITE_ID
+        return site_id
+
+    @model_validator(mode="after")
+    def _check_user_xor_pat_login(self) -> "TableauRevampCredentials":
+        """
+        Checks that credentials are correctly input, it means either:
+        - User and password are filled
+        - Token and Token name are filled
+        """
+        user_login = self.password and self.user
+        pat_login = self.token_name and self.token
+        if not user_login and not pat_login:
+            raise ValueError("Either token or user identification is required")
+        if user_login and pat_login:
+            raise ValueError("Can't have both token and user identification")
+        return self

castor_extractor/warehouse/redshift/queries/column.sql

@@ -28,7 +28,6 @@ information_tables AS (
         i.table_id || '.' || c.column_name AS column_id,
         c.data_type,
         c.ordinal_position,
-        c.column_default,
         c.is_nullable,
         c.character_maximum_length,
         c.character_octet_length,
@@ -59,7 +58,6 @@ raw_tables AS (
         a.attname AS column_name,
         c.oid::TEXT || '.' || a.attname AS column_id,
         a.attnum AS ordinal_position,
-        ad.adsrc AS column_default,
         CASE
             WHEN t.typname = 'bpchar' THEN 'char'
             ELSE t.typname
@@ -90,7 +88,6 @@ tables AS (
         COALESCE(i.data_type, r.data_type) AS data_type,
         COALESCE(i.ordinal_position, r.ordinal_position) AS ordinal_position,
         COALESCE(i.is_nullable, r.is_nullable) AS is_nullable,
-        COALESCE(i.column_default, r.column_default) AS column_default,
         i.character_maximum_length::INT AS character_maximum_length,
         i.character_octet_length::INT AS character_octet_length,
         i.numeric_precision::INT AS numeric_precision,
@@ -117,7 +114,6 @@ views_late_binding AS (
         c.data_type,
         c.ordinal_position,
         'YES' AS is_nullable,
-        NULL::TEXT AS column_default,
         NULL::INT AS character_maximum_length,
         NULL::INT AS character_octet_length,
         NULL::INT AS numeric_precision,
@@ -162,7 +158,6 @@ external_columns AS (
         c.external_type AS data_type,
         MIN(c.columnnum) AS ordinal_position,
         CASE c.is_nullable WHEN 'false' THEN 'NO' ELSE 'YES' END AS is_nullable,
-        NULL AS column_default,
         NULL AS character_maximum_length,
         NULL AS character_octet_length,
         NULL AS numeric_precision,

castor_extractor/warehouse/salesforce/extract.py

@@ -2,7 +2,7 @@ import logging
 from typing import Dict, List, Tuple
 
 from ...utils import AbstractStorage, LocalStorage, write_summary
-from ...utils.salesforce import to_credentials
+from ...utils.salesforce import SalesforceCredentials
 from ..abstract import (
     SupportedAssets,
     WarehouseAsset,
@@ -93,7 +93,7 @@ def extract_all(**kwargs) -> None:
     """
     output_directory, skip_existing = common_args(kwargs)
 
-    client = SalesforceClient(credentials=to_credentials(kwargs))
+    client = SalesforceClient(credentials=SalesforceCredentials(**kwargs))
     storage = LocalStorage(directory=output_directory)
     extractor = SalesforceExtractionProcessor(
         client=client,

castor_extractor/warehouse/snowflake/queries/column.sql

@@ -28,7 +28,6 @@ SELECT
     c.table_catalog_id AS database_id,
     c.table_catalog AS database_name,
     c.ordinal_position,
-    c.column_default,
     c.is_nullable,
     c.data_type,
     c.maximum_cardinality,

castor_extractor/warehouse/synapse/queries/column.sql

@@ -18,7 +18,6 @@ SELECT
     COALESCE(su.uid, pu.uid) AS column_owner_id,
     ty.name AS data_type,
     ty.is_nullable AS is_nullable,
-    sc.column_default AS column_default,
     c.max_length AS character_maximum_length,
     o.create_date AS created_at,
     o.modify_date AS updated_at,

{castor_extractor-0.17.4.dist-info → castor_extractor-0.18.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.17.4
+Version: 0.18.2
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -37,13 +37,13 @@ Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=2,<3)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
-Requires-Dist: looker-sdk (>=23.0.0) ; extra == "looker" or extra == "all"
+Requires-Dist: looker-sdk (>=23.0.0,<24.0.0) ; extra == "looker" or extra == "all"
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
-Requires-Dist: numpy (<1.25) ; python_version >= "3.8" and python_version < "3.9"
-Requires-Dist: numpy (>=1.26,<2) ; python_version >= "3.12" and python_version < "3.13"
-Requires-Dist: pandas (>=2,<2.2.0) ; python_version >= "3.9" and python_full_version <= "3.11.0"
-Requires-Dist: pandas (>=2.0,<2.1) ; python_version >= "3.8" and python_version < "3.9"
-Requires-Dist: pandas (>=2.1,<2.2.0) ; python_version >= "3.12" and python_version < "3.13"
+Requires-Dist: numpy (<1.25) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "bigquery" or extra == "databricks" or extra == "all")
+Requires-Dist: numpy (<2) ; extra == "bigquery" or extra == "databricks" or extra == "all"
+Requires-Dist: numpy (>=1.26) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "bigquery" or extra == "databricks" or extra == "all")
+Requires-Dist: pandas (<2.1) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "databricks" or extra == "all")
+Requires-Dist: pandas (>=2.1) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "databricks" or extra == "all")
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0) ; extra == "metabase" or extra == "postgres" or extra == "redshift" or extra == "all"
 Requires-Dist: pycryptodome (>=3.0.0,<4.0.0) ; extra == "metabase" or extra == "all"
 Requires-Dist: pydantic (>=2.6,<3.0)
@@ -52,13 +52,13 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=69,<70)
+Requires-Dist: setuptools (>=70,<71)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
 Requires-Dist: sqlalchemy-bigquery[bqstorage] (>=1.0.0,<=2.0.0) ; extra == "bigquery" or extra == "all"
 Requires-Dist: sqlalchemy-redshift (>=0.8.14,<0.9.0) ; extra == "redshift" or extra == "all"
-Requires-Dist: tableauserverclient (==0.17.0) ; extra == "tableau" or extra == "all"
+Requires-Dist: tableauserverclient (==0.25.0) ; extra == "tableau" or extra == "all"
 Requires-Dist: tqdm (>=4.0.0,<5.0.0)
 Requires-Dist: typing-extensions (>=4,<5)
 Requires-Dist: websocket-client (>=1,<2) ; extra == "qlik" or extra == "all"