castor-extractor 0.21.3__tar.gz → 0.21.7__tar.gz

{castor_extractor-0.21.3 → castor_extractor-0.21.7}/CHANGELOG.md

@@ -1,6 +1,26 @@
 
 # Changelog
 
+## 0.21.8 - 2024-11-26
+
+* Redshift: improve deduplication of columns
+
+## 0.21.7 - 2024-11-26
+
+* Metabase: stop using deprecated table `view_log`
+
+## 0.21.6 - 2024-11-22
+
+* bump dependencies: ruff, setuptools
+
+## 0.21.5 - 2024-11-20
+
+* PostgreSQL: Fix schema extraction when owner is a role without login privilege
+
+## 0.21.4 - 2024-11-20
+
+* Uploader: Support environment variables as settings
+
 ## 0.21.3 - 2024-11-07
 
 * Tableau: Fix metrics definition url

{castor_extractor-0.21.3 → castor_extractor-0.21.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.21.3
+Version: 0.21.7
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -52,7 +52,7 @@ Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all
 Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
 Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
-Requires-Dist: setuptools (>=74,<75)
+Requires-Dist: setuptools (>=75.3.0,<75.4.0)
 Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
 Requires-Dist: sqlalchemy (>=1.4,<1.5)
@@ -208,6 +208,26 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.21.8 - 2024-11-26
+
+* Redshift: improve deduplication of columns
+
+## 0.21.7 - 2024-11-26
+
+* Metabase: stop using deprecated table `view_log`
+
+## 0.21.6 - 2024-11-22
+
+* bump dependencies: ruff, setuptools
+
+## 0.21.5 - 2024-11-20
+
+* PostgreSQL: Fix schema extraction when owner is a role without login privilege
+
+## 0.21.4 - 2024-11-20
+
+* Uploader: Support environment variables as settings
+
 ## 0.21.3 - 2024-11-07
 
 * Tableau: Fix metrics definition url

castor_extractor-0.21.7/castor_extractor/commands/upload.py

@@ -0,0 +1,48 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.uploader import (  # type: ignore
+    FileType,
+    upload_any,
+)
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def _args() -> ArgumentParser:
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-k",
+        "--token",
+        help="API token provided by CastorDoc",
+    )
+    parser.add_argument(
+        "-s",
+        "--source_id",
+        help="source id provided by CastorDoc",
+    )
+    group = parser.add_mutually_exclusive_group(required=False)
+    group.add_argument("-f", "--file_path", help="path to file to upload")
+    group.add_argument(
+        "-d",
+        "--directory_path",
+        help="""directory containing the files to upload.
+                WARNING: it will upload all the files included
+                in the given repository.""",
+    )
+    supported_file_type = [ft.value for ft in FileType]
+    parser.add_argument(
+        "-t",
+        "--file_type",
+        help="type of file to upload, currently supported are {}".format(
+            supported_file_type,
+        ),
+        choices=supported_file_type,
+    )
+    return parser
+
+
+def main():
+    parser = _args()
+    upload_any(**parse_filled_arguments(parser))

castor_extractor-0.21.7/castor_extractor/uploader/__init__.py

@@ -0,0 +1,2 @@
+from .constant import FileType
+from .upload import upload, upload_any, upload_manifest

castor_extractor-0.21.7/castor_extractor/uploader/settings.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from pydantic import UUID4, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from .constant import FileType
+
+UPLOADER_ENV_PREFIX = "CASTOR_UPLOADER_"
+
+
+class UploaderSettings(BaseSettings):
+    """Class holding Castor uploader attributes"""
+
+    model_config = SettingsConfigDict(
+        env_prefix=UPLOADER_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    directory_path: Optional[str] = None
+    file_path: Optional[str] = None
+    file_type: FileType
+    source_id: UUID4
+    token: str = Field(repr=False)

{castor_extractor-0.21.3 → castor_extractor-0.21.7}/castor_extractor/uploader/upload.py

@@ -17,6 +17,7 @@ from .constant import (
     FileType,
 )
 from .env import get_blob_env
+from .settings import UploaderSettings
 from .utils import iter_files
 
 logger = logging.getLogger(__name__)
@@ -95,13 +96,15 @@ def _upload(
 def upload_manifest(
     token: str,
     source_id: UUID,
-    file_path: str,
+    file_path: Optional[str] = None,
 ) -> None:
     """
-    credentials: path to file or dict
+    token: backend public API token
     source_id: id for the source
     file_path: path to the local manifest to upload
     """
+    if not file_path:
+        raise ValueError("file path is needed to upload a manifest")
     _upload(token, source_id, file_path, FileType.DBT)
 
 
@@ -113,7 +116,7 @@ def upload(
     directory_path: Optional[str] = None,
 ) -> None:
     """
-    credentials: path to file or dict
+    token: backend public API token
     source_id: id for the source
     file_type: type of file(s) uploaded - see FileType Enum
     file_path: path to the local visualization or warehouse file to upload
@@ -130,3 +133,35 @@ def upload(
 
     for file_ in files:
         _upload(token, source_id, file_, file_type)
+
+
+def upload_any(**kwargs) -> None:
+    """
+    entrypath to upload either a file or a manifest
+
+    token: backend public API token
+    source_id: id for the source
+    file_type: type of file(s) uploaded - see FileType Enum
+    file_path: path to the local visualization or warehouse file to upload
+    directory_path: path to the local directory containing files to upload
+    """
+
+    settings = UploaderSettings(**kwargs)
+    file_type = settings.file_type
+
+    if file_type == FileType.DBT:
+        assert not settings.directory_path
+        upload_manifest(
+            token=settings.token,
+            source_id=settings.source_id,
+            file_path=settings.file_path,
+        )
+        return None
+
+    upload(
+        token=settings.token,
+        source_id=settings.source_id,
+        file_type=file_type,
+        file_path=settings.file_path,
+        directory_path=settings.directory_path,
+    )

castor_extractor-0.21.7/castor_extractor/visualization/metabase/client/db/queries/card.sql

@@ -0,0 +1,4 @@
+SELECT
+    *
+FROM
+    {schema}.report_card

castor_extractor-0.21.7/castor_extractor/visualization/metabase/client/db/queries/dashboard.sql

@@ -0,0 +1,4 @@
+SELECT
+    *
+FROM
+    {schema}.report_dashboard

{castor_extractor-0.21.3 → castor_extractor-0.21.7}/castor_extractor/warehouse/postgres/queries/schema.sql

@@ -3,12 +3,12 @@ SELECT
     db.datname AS database_name,
     ns.oid::TEXT AS schema_id,
     ns.nspname AS schema_name,
-    u.usename AS schema_owner,
-    u.usesysid AS schema_owner_id,
+    r.rolname AS schema_owner,
+    ns.nspowner AS schema_owner_id,
     de.description AS "comment"
 FROM pg_catalog.pg_namespace AS ns
     CROSS JOIN pg_catalog.pg_database AS db
-    JOIN pg_catalog.pg_user AS u ON u.usesysid = ns.nspowner
+    LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = ns.nspowner
     LEFT JOIN pg_catalog.pg_description AS de ON de.classoid = ns.oid
 WHERE TRUE
     AND db.datname = CURRENT_DATABASE()

{castor_extractor-0.21.3 → castor_extractor-0.21.7}/castor_extractor/warehouse/postgres/queries/table.sql

@@ -4,14 +4,14 @@ WITH ids AS (
         t.relname AS table_name,
         n.nspname AS schema_name,
         n.oid AS schema_id,
-        u.usename AS table_owner,
+        r.rolname AS table_owner,
         t.relowner AS table_owner_id,
         td.description AS "comment",
         t.reltuples::BIGINT AS tuples
     FROM pg_class AS t
         JOIN pg_catalog.pg_namespace AS n ON n.oid = t.relnamespace
         LEFT JOIN pg_catalog.pg_description AS td ON td.objoid = t.oid AND td.objsubid = 0
-        LEFT JOIN pg_catalog.pg_user AS u ON u.usesysid = t.relowner
+        LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = t.relowner
     WHERE TRUE
         AND n.nspname NOT LIKE 'pg_%%'
         AND n.nspname NOT IN ('catalog_history', 'information_schema')

{castor_extractor-0.21.3 → castor_extractor-0.21.7}/castor_extractor/warehouse/redshift/queries/column.sql

@@ -155,9 +155,9 @@ external_columns AS (
         db.datname || '.' || s.schemaname || '.' || c.tablename AS table_id,
         c.columnname AS column_name,
         db.datname || '.' || s.schemaname || '.' || c.tablename || '.' || c.columnname AS column_id,
-        c.external_type AS data_type,
+        MIN(c.external_type) AS data_type,
         MIN(c.columnnum) AS ordinal_position,
-        CASE c.is_nullable WHEN 'false' THEN 'NO' ELSE 'YES' END AS is_nullable,
+        MIN(CASE c.is_nullable WHEN 'false' THEN 'NO' ELSE 'YES' END) AS is_nullable,
         NULL AS character_maximum_length,
         NULL AS character_octet_length,
         NULL AS numeric_precision,
@@ -172,7 +172,7 @@ external_columns AS (
         JOIN pg_catalog.pg_database AS db ON CURRENT_DATABASE() = db.datname
 
     -- To remove duplicate column names that can occur in external tables (no check on CSVs)
-    GROUP BY database_name, database_id, schema_name, schema_id, table_name, table_id, column_name, column_id, data_type, is_nullable
+    GROUP BY database_name, database_id, schema_name, schema_id, table_name, table_id, column_name, column_id
 )
 
 SELECT * FROM tables

{castor_extractor-0.21.3 → castor_extractor-0.21.7}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.21.3"
+version = "0.21.7"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -51,7 +51,7 @@ python-dateutil = ">=2.0.0, <=3.0.0"
 requests = ">=2.0.0, <3.0.0"
 snowflake-connector-python = { version = "^3.4.0", optional = true }
 snowflake-sqlalchemy = { version = "!=1.2.5, <2.0.0", optional = true }
-setuptools = "^74"
+setuptools = "~75.3.0"  # locked, requires python >= 3.9
 sqlalchemy = "~1.4"
 sqlalchemy-bigquery = { version = ">=1.0.0, <= 2.0.0", optional = true, extras = [
   "bqstorage",
@@ -108,10 +108,10 @@ all = [
 pytest = "^8"
 pytest-freezegun = "^0.4"
 requests-mock = "^1.11"
-ruff = "^0.6.4"
+ruff = "^0.7.4"
 
 [tool.poetry.group.dev.dependencies]
-deptry = "^0.20"
+deptry = "^0.20"  # locked, requires python >= 3.9
 
 [tool.poetry.scripts]
 castor-extract-bigquery = "castor_extractor.commands.extract_bigquery:main"

castor_extractor-0.21.3/castor_extractor/commands/upload.py

@@ -1,68 +0,0 @@
-import argparse
-import logging
-
-from castor_extractor.uploader import (  # type: ignore
-    FileType,
-    upload,
-    upload_manifest,
-)
-
-FILE_TYPES = {FileType.QUALITY, FileType.VIZ, FileType.WAREHOUSE}
-
-logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
-
-
-def _args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-k",
-        "--token",
-        required=True,
-        help="""API token provided by Castor""",
-    )
-    parser.add_argument(
-        "-s",
-        "--source_id",
-        required=True,
-        help="source id provided by castor",
-    )
-    group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument("-f", "--file_path", help="path to file to upload")
-    group.add_argument(
-        "-d",
-        "--directory_path",
-        help="""directory containing the files to upload.
-                WARNING: it will upload all the files included
-                in the given repository.""",
-    )
-    supported_file_type = [ft.value for ft in FileType]
-    parser.add_argument(
-        "-t",
-        "--file_type",
-        help="type of file to upload, currently supported are {}".format(
-            supported_file_type,
-        ),
-        choices=supported_file_type,
-    )
-    parsed = parser.parse_args()
-    return {
-        "token": parsed.token,
-        "source_id": parsed.source_id,
-        "file_path": parsed.file_path,
-        "directory_path": parsed.directory_path,
-        "file_type": FileType(parsed.file_type),
-    }
-
-
-def main():
-    params = _args()
-
-    file_type = params.get("file_type")
-    if file_type in FILE_TYPES:
-        upload(**params)
-
-    if file_type == FileType.DBT:
-        dir_path = params.pop("directory_path")
-        assert not dir_path
-        del params["file_type"]
-        upload_manifest(**params)

castor_extractor-0.21.3/castor_extractor/uploader/__init__.py

@@ -1,2 +0,0 @@
-from .constant import FileType
-from .upload import upload, upload_manifest

castor_extractor-0.21.3/castor_extractor/visualization/metabase/client/db/queries/card.sql

@@ -1,21 +0,0 @@
-WITH card_view_log AS
-(
-    SELECT
-        model_id,
-        COUNT(id) AS view_count,
-        MAX("timestamp") AS last_viewed_at
-    FROM
-        {schema}.view_log
-    WHERE
-        model = 'card'
-    GROUP BY
-        1
-)
-
-SELECT
-    rc.*,
-    vl.last_viewed_at AS last_viewed_at,
-    COALESCE(vl.view_count, 0) AS view_count
-FROM
-    {schema}.report_card AS rc
-    LEFT JOIN card_view_log AS vl ON rc.id = vl.model_id

castor_extractor-0.21.3/castor_extractor/visualization/metabase/client/db/queries/dashboard.sql

@@ -1,21 +0,0 @@
-WITH dashboard_view_log AS
-(
-    SELECT
-        model_id,
-        COUNT(id) AS view_count,
-        MAX("timestamp") AS last_viewed_at
-    FROM
-        {schema}.view_log
-    WHERE
-        model = 'dashboard'
-    GROUP BY
-        1
-)
-
-SELECT
-    rd.*,
-    vl.last_viewed_at AS last_viewed_at,
-    COALESCE(vl.view_count, 0) AS view_count
-FROM
-    {schema}.report_dashboard AS rd
-    LEFT JOIN dashboard_view_log AS vl ON rd.id = vl.model_id