castor-extractor 0.21.1__py3-none-any.whl → 0.21.5__py3-none-any.whl

CHANGELOG.md

@@ -1,6 +1,22 @@
 
 # Changelog
 
+## 0.21.5 - 2024-11-20
+
+* PostgreSQL: Fix schema extraction when owner is a role without login privilege
+
+## 0.21.4 - 2024-11-20
+
+* Uploader: Support environment variables as settings
+
+## 0.21.3 - 2024-11-07
+
+* Tableau: Fix metrics definition url
+
+## 0.21.2 - 2024-11-06
+
+* Adding fetch method for confluence client
+
 ## 0.21.1 - 2024-10-23
 
 * Warning message to deprecate python < 3.9

castor_extractor/commands/upload.py

@@ -1,32 +1,28 @@
-import argparse
 import logging
+from argparse import ArgumentParser
 
 from castor_extractor.uploader import (  # type: ignore
     FileType,
-    upload,
-    upload_manifest,
+    upload_any,
 )
-
-FILE_TYPES = {FileType.QUALITY, FileType.VIZ, FileType.WAREHOUSE}
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
 
 
-def _args():
-    parser = argparse.ArgumentParser()
+def _args() -> ArgumentParser:
+    parser = ArgumentParser()
     parser.add_argument(
         "-k",
         "--token",
-        required=True,
-        help="""API token provided by Castor""",
+        help="API token provided by CastorDoc",
     )
     parser.add_argument(
         "-s",
         "--source_id",
-        required=True,
-        help="source id provided by castor",
+        help="source id provided by CastorDoc",
     )
-    group = parser.add_mutually_exclusive_group(required=True)
+    group = parser.add_mutually_exclusive_group(required=False)
     group.add_argument("-f", "--file_path", help="path to file to upload")
     group.add_argument(
         "-d",
@@ -44,25 +40,9 @@ def _args():
         ),
         choices=supported_file_type,
     )
-    parsed = parser.parse_args()
-    return {
-        "token": parsed.token,
-        "source_id": parsed.source_id,
-        "file_path": parsed.file_path,
-        "directory_path": parsed.directory_path,
-        "file_type": FileType(parsed.file_type),
-    }
+    return parser
 
 
 def main():
-    params = _args()
-
-    file_type = params.get("file_type")
-    if file_type in FILE_TYPES:
-        upload(**params)
-
-    if file_type == FileType.DBT:
-        dir_path = params.pop("directory_path")
-        assert not dir_path
-        del params["file_type"]
-        upload_manifest(**params)
+    parser = _args()
+    upload_any(**parse_filled_arguments(parser))

castor_extractor/knowledge/confluence/client/client.py

@@ -1,6 +1,6 @@
 from functools import partial
 from http import HTTPStatus
-from typing import Optional
+from typing import Iterator, Optional
 
 from ....utils import (
     APIClient,
@@ -8,6 +8,9 @@ from ....utils import (
     RequestSafeMode,
     fetch_all_pages,
 )
+from ..assets import (
+    ConfluenceAsset,
+)
 from .credentials import ConfluenceCredentials
 from .endpoints import ConfluenceEndpointFactory
 from .pagination import ConfluencePagination
@@ -58,3 +61,14 @@ class ConfluenceClient(APIClient):
             data=request_body,
         )
         yield from fetch_all_pages(request, ConfluencePagination)
+
+    def fetch(self, asset: ConfluenceAsset) -> Iterator[dict]:
+        """Returns the needed metadata for the queried asset"""
+        if asset == ConfluenceAsset.PAGES:
+            yield from self.pages()
+
+        elif asset == ConfluenceAsset.USERS:
+            yield from self.users()
+
+        else:
+            raise ValueError(f"This asset {asset} is unknown")

castor_extractor/uploader/__init__.py

@@ -1,2 +1,2 @@
 from .constant import FileType
-from .upload import upload, upload_manifest
+from .upload import upload, upload_any, upload_manifest

castor_extractor/uploader/settings.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from pydantic import UUID4, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from .constant import FileType
+
+UPLOADER_ENV_PREFIX = "CASTOR_UPLOADER_"
+
+
+class UploaderSettings(BaseSettings):
+    """Class holding Castor uploader attributes"""
+
+    model_config = SettingsConfigDict(
+        env_prefix=UPLOADER_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    directory_path: Optional[str] = None
+    file_path: Optional[str] = None
+    file_type: FileType
+    source_id: UUID4
+    token: str = Field(repr=False)

castor_extractor/uploader/upload.py

@@ -17,6 +17,7 @@ from .constant import (
     FileType,
 )
 from .env import get_blob_env
+from .settings import UploaderSettings
 from .utils import iter_files
 
 logger = logging.getLogger(__name__)
@@ -95,13 +96,15 @@ def _upload(
 def upload_manifest(
     token: str,
     source_id: UUID,
-    file_path: str,
+    file_path: Optional[str] = None,
 ) -> None:
     """
-    credentials: path to file or dict
+    token: backend public API token
     source_id: id for the source
     file_path: path to the local manifest to upload
     """
+    if not file_path:
+        raise ValueError("file path is needed to upload a manifest")
     _upload(token, source_id, file_path, FileType.DBT)
 
 
@@ -113,7 +116,7 @@ def upload(
     directory_path: Optional[str] = None,
 ) -> None:
     """
-    credentials: path to file or dict
+    token: backend public API token
     source_id: id for the source
     file_type: type of file(s) uploaded - see FileType Enum
     file_path: path to the local visualization or warehouse file to upload
@@ -130,3 +133,35 @@ def upload(
 
     for file_ in files:
         _upload(token, source_id, file_, file_type)
+
+
+def upload_any(**kwargs) -> None:
+    """
+    entrypath to upload either a file or a manifest
+
+    token: backend public API token
+    source_id: id for the source
+    file_type: type of file(s) uploaded - see FileType Enum
+    file_path: path to the local visualization or warehouse file to upload
+    directory_path: path to the local directory containing files to upload
+    """
+
+    settings = UploaderSettings(**kwargs)
+    file_type = settings.file_type
+
+    if file_type == FileType.DBT:
+        assert not settings.directory_path
+        upload_manifest(
+            token=settings.token,
+            source_id=settings.source_id,
+            file_path=settings.file_path,
+        )
+        return None
+
+    upload(
+        token=settings.token,
+        source_id=settings.source_id,
+        file_type=file_type,
+        file_path=settings.file_path,
+        directory_path=settings.directory_path,
+    )

castor_extractor/visualization/tableau_revamp/client/client_rest_api.py

@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__)
 
 _PULSE_API = "api/-/pulse"
 
-_METRICS_URL = "{base}/pulse/site/{site}/metrics/{definition_id}"
+_METRICS_DEFINITION_URL = "{base}/pulse/site/{site}/{definition_id}"
 
 
 def _pick(
@@ -89,7 +89,7 @@ class TableauClientRestApi:
         site = self._get_site_name()
         base_url = self._server.server_address.strip("/")
         for row in data:
-            row["metadata"]["url"] = _METRICS_URL.format(
+            row["metadata"]["url"] = _METRICS_DEFINITION_URL.format(
                 base=base_url,
                 site=site,
                 definition_id=row["metadata"]["id"],

castor_extractor/warehouse/postgres/queries/schema.sql

@@ -3,12 +3,12 @@ SELECT
     db.datname AS database_name,
     ns.oid::TEXT AS schema_id,
     ns.nspname AS schema_name,
-    u.usename AS schema_owner,
-    u.usesysid AS schema_owner_id,
+    r.rolname AS schema_owner,
+    ns.nspowner AS schema_owner_id,
     de.description AS "comment"
 FROM pg_catalog.pg_namespace AS ns
     CROSS JOIN pg_catalog.pg_database AS db
-    JOIN pg_catalog.pg_user AS u ON u.usesysid = ns.nspowner
+    LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = ns.nspowner
     LEFT JOIN pg_catalog.pg_description AS de ON de.classoid = ns.oid
 WHERE TRUE
     AND db.datname = CURRENT_DATABASE()

castor_extractor/warehouse/postgres/queries/table.sql

@@ -4,14 +4,14 @@ WITH ids AS (
         t.relname AS table_name,
         n.nspname AS schema_name,
         n.oid AS schema_id,
-        u.usename AS table_owner,
+        r.rolname AS table_owner,
         t.relowner AS table_owner_id,
         td.description AS "comment",
         t.reltuples::BIGINT AS tuples
     FROM pg_class AS t
         JOIN pg_catalog.pg_namespace AS n ON n.oid = t.relnamespace
         LEFT JOIN pg_catalog.pg_description AS td ON td.objoid = t.oid AND td.objsubid = 0
-        LEFT JOIN pg_catalog.pg_user AS u ON u.usesysid = t.relowner
+        LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = t.relowner
     WHERE TRUE
         AND n.nspname NOT LIKE 'pg_%%'
         AND n.nspname NOT IN ('catalog_history', 'information_schema')

{castor_extractor-0.21.1.dist-info → castor_extractor-0.21.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.21.1
+Version: 0.21.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -208,6 +208,22 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.21.5 - 2024-11-20
+
+* PostgreSQL: Fix schema extraction when owner is a role without login privilege
+
+## 0.21.4 - 2024-11-20
+
+* Uploader: Support environment variables as settings
+
+## 0.21.3 - 2024-11-07
+
+* Tableau: Fix metrics definition url
+
+## 0.21.2 - 2024-11-06
+
+* Adding fetch method for confluence client
+
 ## 0.21.1 - 2024-10-23
 
 * Warning message to deprecate python < 3.9

{castor_extractor-0.21.1.dist-info → castor_extractor-0.21.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=U0folo6uva597R9IArkQzRau3UldJIdvq99aWOEWKkA,14355
+CHANGELOG.md,sha256=HxssLs5JkyCp21TZ3l85OSMkFvaO5mrr3pZRXM-EqUg,14672
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -27,7 +27,7 @@ castor_extractor/commands/extract_sqlserver.py,sha256=lwhbcNChaXHZgMgSOch3faVr7W
 castor_extractor/commands/extract_tableau.py,sha256=VUb_1Y85EzfF1f9OaCQQt8kFYBdp0u31Mw1Wm2fkxWs,1221
 castor_extractor/commands/extract_thoughtspot.py,sha256=caAYJlH-vK7u5IUB6OKXxcaWfLgc7d_XqnFDWK6YNS4,639
 castor_extractor/commands/file_check.py,sha256=VSD84kpQKf7b0wJOhUgkJQ9n4mK3v52sjMWL7wkNYa0,2667
-castor_extractor/commands/upload.py,sha256=WLDI3zDmK2CjtbxiMWX2mZGjxx8DozfCw6tLE3CAMcE,1833
+castor_extractor/commands/upload.py,sha256=rLXp7gQ8zb1kLbho4FT87q8eJd8Gvo_TkyIynAaQ-4s,1342
 castor_extractor/file_checker/__init__.py,sha256=OSt6YLhUT42U_Cp3LCLHMVruwDkksL75Ij13X2UPnVk,119
 castor_extractor/file_checker/column.py,sha256=fMchy5v-Sd-0xuYS0V9mob7wnljslzWLhQGqrKGybdk,3097
 castor_extractor/file_checker/column_test.py,sha256=1j8PxvmvmJgpd-mk30iMYOme32ovPSIn4yCXywFoXrg,1935
@@ -43,7 +43,7 @@ castor_extractor/knowledge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 castor_extractor/knowledge/confluence/__init__.py,sha256=pRT615pMDlB7Ifs09erVn2EdpZHgkvX5selemWU3VPE,129
 castor_extractor/knowledge/confluence/assets.py,sha256=zv2G2LB8H0fKDbVJ4kHrAjbqehXI_K-wgd_ghSXGFvs,144
 castor_extractor/knowledge/confluence/client/__init__.py,sha256=ALAzo0JEhxFzH2FnIO6HmtkAGS2_bGY8KXXMcTGV3aE,84
-castor_extractor/knowledge/confluence/client/client.py,sha256=ZgAUlEGwpSN1QZAFGpU-7gob_wWI3wpBh1iTycxUZc4,1667
+castor_extractor/knowledge/confluence/client/client.py,sha256=ZBabhxlaatbfT19ijGTmF0UJdWSANz7_q5IU7AmB4PM,2090
 castor_extractor/knowledge/confluence/client/credentials.py,sha256=2LUjnCOMwq2NBrkHty99TpWlgyOOsYjwC9NeekZMH84,422
 castor_extractor/knowledge/confluence/client/endpoints.py,sha256=b7PIvw9_W942L4zsEZa__KhZDTo4yt-CdIO0eas69TE,736
 castor_extractor/knowledge/confluence/client/pagination.py,sha256=ty4meiMEujDVSiQyOJTibd-ReYyDyGezdFuk7EAGtMA,862
@@ -68,11 +68,12 @@ castor_extractor/quality/soda/client/credentials.py,sha256=R1g7nHpJlQ5hBjtUFN06Q
 castor_extractor/quality/soda/client/endpoints.py,sha256=x3B-XlnDF8NJMuk-81N72_6HA-YZEzA895khLyj0j54,228
 castor_extractor/quality/soda/client/pagination.py,sha256=IrAnju5CFipHarTLZRWZflVZ3KHoRVMvaCaNAePHVug,555
 castor_extractor/types.py,sha256=Hd_shbsGAknJLTrAk3SBxZeFPOlbWBXXjIscC9C7CW8,1281
-castor_extractor/uploader/__init__.py,sha256=SSRtwjg-dNoxME-RJy9G1flASiUKAC5bH1htq3CURQg,75
+castor_extractor/uploader/__init__.py,sha256=A4bq_SrEtKAsl0r_D_duSTvL5WIQjVfsMy7tDx9IKg0,87
 castor_extractor/uploader/constant.py,sha256=yTigLHDlYwoRr6CpFIl7ReElFsQd4H-qkluMZJPWSx0,865
 castor_extractor/uploader/env.py,sha256=5HSniVSOYVg4u38O4k8TB_qaJq9s8yJ1hjedkq_gdVg,878
 castor_extractor/uploader/env_test.py,sha256=ClCWWtwd2N-5ClIDUxVMeKkWfhhOTxpppsXUDmdjxSg,472
-castor_extractor/uploader/upload.py,sha256=c86NP4ZxWnz3Hy1iWDYd9qjJSSjZ1bLq3fxVGBIU4Rc,3238
+castor_extractor/uploader/settings.py,sha256=3MvOX-UFRqrLZoiT7wYn9jUGro7NX4RCafYzrXrLQtA,590
+castor_extractor/uploader/upload.py,sha256=XGoqr4UdpsARyLG0Mp5enBqDvc6m_WCK9nMB3yhUwlM,4296
 castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
 castor_extractor/uploader/utils.py,sha256=Tx_i875L2vJ8btOLV3-L0UMEFiyhH8E5n0XXRyLjO0Y,793
 castor_extractor/utils/__init__.py,sha256=jyYquzC2-R-UYl3VTP49ZDHB0IErGogTPMy3GfScbaA,1524
@@ -285,7 +286,7 @@ castor_extractor/visualization/tableau_revamp/assets.py,sha256=8sJsK6Qixao6xVmVa
 castor_extractor/visualization/tableau_revamp/client/__init__.py,sha256=wmS9uLtUiqNYVloi0-DgD8d2qzu3RVZEAtWiaDp6G_M,90
 castor_extractor/visualization/tableau_revamp/client/client.py,sha256=oaxvPsCccAcTWooXmDQNcJ6RFUVsCUzl6HxaHIwh5kU,7564
 castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py,sha256=yNnGR3Tk32TUmaDejaz5fkw2p9DtmMeCv5rsZNOHUfY,3047
-castor_extractor/visualization/tableau_revamp/client/client_rest_api.py,sha256=0g8AddrhzirRCUWR2jrudPR02mk4Of5YilWth7zJO-g,4016
+castor_extractor/visualization/tableau_revamp/client/client_rest_api.py,sha256=HZ6KdNJ6sqhWElfpYlUwbZDJXxoKkNc3p-YAKExzDxM,4030
 castor_extractor/visualization/tableau_revamp/client/client_tsc.py,sha256=BBwIOqK2zU66udFRmLGmB_3J1ILGhVOY5Hq4nmsonF0,1853
 castor_extractor/visualization/tableau_revamp/client/credentials.py,sha256=qA-EaX-4rbQRsn8v4zWh5Kh784ndHLjJaoZwnkQgCyo,1905
 castor_extractor/visualization/tableau_revamp/client/errors.py,sha256=dTe1shqmWmAXpDpCz-E24m8dGYjt6rvIGV9qQb4jnvI,150
@@ -364,8 +365,8 @@ castor_extractor/warehouse/postgres/queries/.sqlfluff,sha256=Z7nq1jdnKxpOKpKTpJI
 castor_extractor/warehouse/postgres/queries/column.sql,sha256=N7UzWK0aslvoaXQritSf-EE7h8fpadT4AYFaCwfMo9s,1632
 castor_extractor/warehouse/postgres/queries/database.sql,sha256=GRM-CHI3EGCjpEEvgzdefLhV74EPTqCOCaud0S1Rv_E,241
 castor_extractor/warehouse/postgres/queries/group.sql,sha256=8p0wlqllnwOTiAgiV237DvFYHGOEcYwaHdyqVQg3F6E,101
-castor_extractor/warehouse/postgres/queries/schema.sql,sha256=cd_bx8uasswxtEPNOuHeN49YBcbeKDvl4yULbpXkEfE,592
-castor_extractor/warehouse/postgres/queries/table.sql,sha256=ok9IGlX8b2v8AETpgG9ns-wfOlnyxz73enEd6tVOLGw,1489
+castor_extractor/warehouse/postgres/queries/schema.sql,sha256=brzNqvkoARjOgXUSu1nvCIXueS6QRk3ozQr_OoRwJdY,594
+castor_extractor/warehouse/postgres/queries/table.sql,sha256=eHiajIJnBj43wpx19fYpGbbI_zvp3-IhqwUv9aVohLo,1485
 castor_extractor/warehouse/postgres/queries/user.sql,sha256=sEXveJAuNvZacvpI6WfwsX6VavoMb2VqYA32f6Dt-_Y,170
 castor_extractor/warehouse/postgres/query.py,sha256=5QmI79BP_EjqxeABNg56rxuM9XuuXHuFFzCk6svnFEw,540
 castor_extractor/warehouse/redshift/__init__.py,sha256=CC82SejYDlwYhZhhn40ln-oTsRx7AJ1Km61cxPkymjE,125
@@ -424,8 +425,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=j_d5-HMnzBouwGfywVZMRSSwbXzPvzDWlFCZmvxcoGQ,539
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.21.1.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.21.1.dist-info/METADATA,sha256=28gV2BAN9hz_TdzFHMxezHikWiOLwTczn0BwVKGrCy0,21575
-castor_extractor-0.21.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.21.1.dist-info/entry_points.txt,sha256=7aVSxc-_2dicp28Ow-S4y0p4wGoTm9zGmVptMvfLdw8,1649
-castor_extractor-0.21.1.dist-info/RECORD,,
+castor_extractor-0.21.5.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.21.5.dist-info/METADATA,sha256=-LHZIKLW0VI1evr0xohzYnR1CBZomKpv_a_khkkShcA,21892
+castor_extractor-0.21.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.21.5.dist-info/entry_points.txt,sha256=7aVSxc-_2dicp28Ow-S4y0p4wGoTm9zGmVptMvfLdw8,1649
+castor_extractor-0.21.5.dist-info/RECORD,,