castor-extractor 0.21.3__tar.gz → 0.21.5__tar.gz

{castor_extractor-0.21.3 → castor_extractor-0.21.5}/CHANGELOG.md

@@ -1,6 +1,14 @@
 
 # Changelog
 
+## 0.21.5 - 2024-11-20
+
+* PostgreSQL: Fix schema extraction when owner is a role without login privilege
+
+## 0.21.4 - 2024-11-20
+
+* Uploader: Support environment variables as settings
+
 ## 0.21.3 - 2024-11-07
 
 * Tableau: Fix metrics definition url

{castor_extractor-0.21.3 → castor_extractor-0.21.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.21.3
+Version: 0.21.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -208,6 +208,14 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.21.5 - 2024-11-20
+
+* PostgreSQL: Fix schema extraction when owner is a role without login privilege
+
+## 0.21.4 - 2024-11-20
+
+* Uploader: Support environment variables as settings
+
 ## 0.21.3 - 2024-11-07
 
 * Tableau: Fix metrics definition url

castor_extractor-0.21.5/castor_extractor/commands/upload.py

@@ -0,0 +1,48 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.uploader import (  # type: ignore
+    FileType,
+    upload_any,
+)
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def _args() -> ArgumentParser:
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-k",
+        "--token",
+        help="API token provided by CastorDoc",
+    )
+    parser.add_argument(
+        "-s",
+        "--source_id",
+        help="source id provided by CastorDoc",
+    )
+    group = parser.add_mutually_exclusive_group(required=False)
+    group.add_argument("-f", "--file_path", help="path to file to upload")
+    group.add_argument(
+        "-d",
+        "--directory_path",
+        help="""directory containing the files to upload.
+                WARNING: it will upload all the files included
+                in the given repository.""",
+    )
+    supported_file_type = [ft.value for ft in FileType]
+    parser.add_argument(
+        "-t",
+        "--file_type",
+        help="type of file to upload, currently supported are {}".format(
+            supported_file_type,
+        ),
+        choices=supported_file_type,
+    )
+    return parser
+
+
+def main():
+    parser = _args()
+    upload_any(**parse_filled_arguments(parser))

castor_extractor-0.21.5/castor_extractor/uploader/__init__.py

@@ -0,0 +1,2 @@
+from .constant import FileType
+from .upload import upload, upload_any, upload_manifest

castor_extractor-0.21.5/castor_extractor/uploader/settings.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from pydantic import UUID4, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from .constant import FileType
+
+UPLOADER_ENV_PREFIX = "CASTOR_UPLOADER_"
+
+
+class UploaderSettings(BaseSettings):
+    """Class holding Castor uploader attributes"""
+
+    model_config = SettingsConfigDict(
+        env_prefix=UPLOADER_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    directory_path: Optional[str] = None
+    file_path: Optional[str] = None
+    file_type: FileType
+    source_id: UUID4
+    token: str = Field(repr=False)

{castor_extractor-0.21.3 → castor_extractor-0.21.5}/castor_extractor/uploader/upload.py

@@ -17,6 +17,7 @@ from .constant import (
     FileType,
 )
 from .env import get_blob_env
+from .settings import UploaderSettings
 from .utils import iter_files
 
 logger = logging.getLogger(__name__)
@@ -95,13 +96,15 @@ def _upload(
 def upload_manifest(
     token: str,
     source_id: UUID,
-    file_path: str,
+    file_path: Optional[str] = None,
 ) -> None:
     """
-    credentials: path to file or dict
+    token: backend public API token
     source_id: id for the source
     file_path: path to the local manifest to upload
     """
+    if not file_path:
+        raise ValueError("file path is needed to upload a manifest")
     _upload(token, source_id, file_path, FileType.DBT)
 
 
@@ -113,7 +116,7 @@ def upload(
     directory_path: Optional[str] = None,
 ) -> None:
     """
-    credentials: path to file or dict
+    token: backend public API token
     source_id: id for the source
     file_type: type of file(s) uploaded - see FileType Enum
     file_path: path to the local visualization or warehouse file to upload
@@ -130,3 +133,35 @@ def upload(
 
     for file_ in files:
         _upload(token, source_id, file_, file_type)
+
+
+def upload_any(**kwargs) -> None:
+    """
+    entrypath to upload either a file or a manifest
+
+    token: backend public API token
+    source_id: id for the source
+    file_type: type of file(s) uploaded - see FileType Enum
+    file_path: path to the local visualization or warehouse file to upload
+    directory_path: path to the local directory containing files to upload
+    """
+
+    settings = UploaderSettings(**kwargs)
+    file_type = settings.file_type
+
+    if file_type == FileType.DBT:
+        assert not settings.directory_path
+        upload_manifest(
+            token=settings.token,
+            source_id=settings.source_id,
+            file_path=settings.file_path,
+        )
+        return None
+
+    upload(
+        token=settings.token,
+        source_id=settings.source_id,
+        file_type=file_type,
+        file_path=settings.file_path,
+        directory_path=settings.directory_path,
+    )

{castor_extractor-0.21.3 → castor_extractor-0.21.5}/castor_extractor/warehouse/postgres/queries/schema.sql

@@ -3,12 +3,12 @@ SELECT
     db.datname AS database_name,
     ns.oid::TEXT AS schema_id,
     ns.nspname AS schema_name,
-    u.usename AS schema_owner,
-    u.usesysid AS schema_owner_id,
+    r.rolname AS schema_owner,
+    ns.nspowner AS schema_owner_id,
     de.description AS "comment"
 FROM pg_catalog.pg_namespace AS ns
     CROSS JOIN pg_catalog.pg_database AS db
-    JOIN pg_catalog.pg_user AS u ON u.usesysid = ns.nspowner
+    LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = ns.nspowner
     LEFT JOIN pg_catalog.pg_description AS de ON de.classoid = ns.oid
 WHERE TRUE
     AND db.datname = CURRENT_DATABASE()

{castor_extractor-0.21.3 → castor_extractor-0.21.5}/castor_extractor/warehouse/postgres/queries/table.sql

@@ -4,14 +4,14 @@ WITH ids AS (
         t.relname AS table_name,
         n.nspname AS schema_name,
         n.oid AS schema_id,
-        u.usename AS table_owner,
+        r.rolname AS table_owner,
         t.relowner AS table_owner_id,
         td.description AS "comment",
         t.reltuples::BIGINT AS tuples
     FROM pg_class AS t
         JOIN pg_catalog.pg_namespace AS n ON n.oid = t.relnamespace
         LEFT JOIN pg_catalog.pg_description AS td ON td.objoid = t.oid AND td.objsubid = 0
-        LEFT JOIN pg_catalog.pg_user AS u ON u.usesysid = t.relowner
+        LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = t.relowner
     WHERE TRUE
         AND n.nspname NOT LIKE 'pg_%%'
         AND n.nspname NOT IN ('catalog_history', 'information_schema')

{castor_extractor-0.21.3 → castor_extractor-0.21.5}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.21.3"
+version = "0.21.5"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.21.3/castor_extractor/commands/upload.py

@@ -1,68 +0,0 @@
-import argparse
-import logging
-
-from castor_extractor.uploader import (  # type: ignore
-    FileType,
-    upload,
-    upload_manifest,
-)
-
-FILE_TYPES = {FileType.QUALITY, FileType.VIZ, FileType.WAREHOUSE}
-
-logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
-
-
-def _args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-k",
-        "--token",
-        required=True,
-        help="""API token provided by Castor""",
-    )
-    parser.add_argument(
-        "-s",
-        "--source_id",
-        required=True,
-        help="source id provided by castor",
-    )
-    group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument("-f", "--file_path", help="path to file to upload")
-    group.add_argument(
-        "-d",
-        "--directory_path",
-        help="""directory containing the files to upload.
-                WARNING: it will upload all the files included
-                in the given repository.""",
-    )
-    supported_file_type = [ft.value for ft in FileType]
-    parser.add_argument(
-        "-t",
-        "--file_type",
-        help="type of file to upload, currently supported are {}".format(
-            supported_file_type,
-        ),
-        choices=supported_file_type,
-    )
-    parsed = parser.parse_args()
-    return {
-        "token": parsed.token,
-        "source_id": parsed.source_id,
-        "file_path": parsed.file_path,
-        "directory_path": parsed.directory_path,
-        "file_type": FileType(parsed.file_type),
-    }
-
-
-def main():
-    params = _args()
-
-    file_type = params.get("file_type")
-    if file_type in FILE_TYPES:
-        upload(**params)
-
-    if file_type == FileType.DBT:
-        dir_path = params.pop("directory_path")
-        assert not dir_path
-        del params["file_type"]
-        upload_manifest(**params)

castor_extractor-0.21.3/castor_extractor/uploader/__init__.py

@@ -1,2 +0,0 @@
-from .constant import FileType
-from .upload import upload, upload_manifest