castor-extractor 0.20.6__py3-none-any.whl → 0.21.1__py3-none-any.whl

CHANGELOG.md

@@ -1,6 +1,22 @@
 
 # Changelog
 
+## 0.21.1 - 2024-10-23
+
+* Warning message to deprecate python < 3.9
+
+## 0.21.0 - 2024-10-23
+
+* Confluence: Added Confluence extractor
+
+## 0.20.8 - 2024-10-19
+
+* bump dependencies (minor and patches)
+
+## 0.20.7 - 2024-10-18
+
+* Metabase: fix `require_ssl` type in credentials
+
 ## 0.20.6 - 2024-10-15
 
 * Tableau: include `site_id` in **workbooks** to build url

castor_extractor/commands/__init__.py

@@ -0,0 +1,3 @@
+from castor_extractor.utils import deprecate_python  # type: ignore
+
+deprecate_python(min_version_supported=(3, 9))

castor_extractor/commands/extract_confluence.py

@@ -0,0 +1,19 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.knowledge import confluence  # type: ignore
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def main():
+    parser = ArgumentParser()
+
+    parser.add_argument("-a", "--account_id", help="Confluence account id")
+    parser.add_argument("-b", "--base_url", help="Confluence account base url")
+    parser.add_argument("-o", "--output", help="Directory to write to")
+    parser.add_argument("-p", "--password", help="Confluence password")
+    parser.add_argument("-u", "--username", help="Confluence username")
+
+    confluence.extract_all(**parse_filled_arguments(parser))

castor_extractor/knowledge/confluence/__init__.py

@@ -0,0 +1,3 @@
+from .assets import ConfluenceAsset
+from .client import ConfluenceClient, ConfluenceCredentials
+from .extract import extract_all

castor_extractor/knowledge/confluence/assets.py

@@ -0,0 +1,8 @@
+from ...types import ExternalAsset
+
+
+class ConfluenceAsset(ExternalAsset):
+    """Confluence assets"""
+
+    PAGES = "pages"
+    USERS = "users"

castor_extractor/knowledge/confluence/client/__init__.py

@@ -0,0 +1,2 @@
+from .client import ConfluenceClient
+from .credentials import ConfluenceCredentials

castor_extractor/knowledge/confluence/client/client.py

@@ -0,0 +1,60 @@
+from functools import partial
+from http import HTTPStatus
+from typing import Optional
+
+from ....utils import (
+    APIClient,
+    BasicAuth,
+    RequestSafeMode,
+    fetch_all_pages,
+)
+from .credentials import ConfluenceCredentials
+from .endpoints import ConfluenceEndpointFactory
+from .pagination import ConfluencePagination
+
+_HEADERS = {
+    "Accept": "application/json",
+    "Content-Type": "application/json",
+}
+
+_MAX_ERROR_IGNORED_COUNT = 10
+_IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY,)
+_SAFE_MODE = RequestSafeMode(
+    max_errors=_MAX_ERROR_IGNORED_COUNT,
+    status_codes=_IGNORED_ERROR_CODES,
+)
+
+
+class ConfluenceClient(APIClient):
+    def __init__(
+        self,
+        credentials: ConfluenceCredentials,
+        safe_mode: Optional[RequestSafeMode] = None,
+    ):
+        self.account_id = credentials.account_id
+        auth = BasicAuth(
+            username=credentials.username, password=credentials.password
+        )
+        super().__init__(
+            auth=auth,
+            host=credentials.base_url,
+            headers=_HEADERS,
+            safe_mode=safe_mode or _SAFE_MODE,
+        )
+
+    def pages(self):
+        request = partial(
+            self._get,
+            endpoint=ConfluenceEndpointFactory.pages(),
+            params={"body-format": "atlas_doc_format"},
+        )
+        yield from fetch_all_pages(request, ConfluencePagination)
+
+    def users(self):
+        request_body = {"accountIds": [self.account_id]}
+        request = partial(
+            self._post,
+            endpoint=ConfluenceEndpointFactory.users(),
+            data=request_body,
+        )
+        yield from fetch_all_pages(request, ConfluencePagination)

castor_extractor/knowledge/confluence/client/credentials.py

@@ -0,0 +1,18 @@
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+CASTOR_ENV_PREFIX = "CASTOR_CONFLUENCE_"
+
+
+class ConfluenceCredentials(BaseSettings):
+    """Class to handle Confluence rest API permissions"""
+
+    model_config = SettingsConfigDict(
+        env_prefix=CASTOR_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    account_id: str
+    base_url: str
+    password: str
+    username: str

castor_extractor/knowledge/confluence/client/endpoints.py

@@ -0,0 +1,25 @@
+class ConfluenceEndpointFactory:
+    """
+    Confluence rest api v2 endpoint factory.
+    https://developer.atlassian.com/cloud/confluence/rest/v2/intro/#about
+    """
+
+    API = "wiki/api/v2/"
+    PAGES = "pages"
+    USERS = "users-bulk"
+
+    @classmethod
+    def pages(cls) -> str:
+        """
+        Endpoint to fetch all pages.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-pages-get
+        """
+        return f"{cls.API}{cls.PAGES}"
+
+    @classmethod
+    def users(cls) -> str:
+        """
+        Endpoint to fetch all user.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-user/#api-users-bulk-post
+        """
+        return f"{cls.API}{cls.USERS}"

castor_extractor/knowledge/confluence/client/pagination.py

@@ -0,0 +1,32 @@
+from typing import Optional, Union
+
+from pydantic import AliasPath, Field
+
+from ....utils import (
+    FetchNextPageBy,
+    PaginationModel,
+)
+
+
+class ConfluencePagination(PaginationModel):
+    """Class to handle paginated results for confluence"""
+
+    fetch_by: FetchNextPageBy = FetchNextPageBy.URL
+
+    results: list = Field(default_factory=list)
+    next_url: Optional[str] = Field(
+        validation_alias=AliasPath("_links", "next"),
+        default=None,
+    )
+
+    def is_last(self) -> bool:
+        """Stopping condition for the pagination"""
+        return self.next_url is None
+
+    def next_page_payload(self) -> Union[str, None]:
+        """Payload enabling to generate the request for the next page"""
+        return self.next_url
+
+    def page_results(self) -> list:
+        """List of results of the current page"""
+        return self.results

castor_extractor/knowledge/confluence/extract.py

@@ -0,0 +1,54 @@
+import logging
+from typing import Iterable, Iterator, Tuple, Union
+
+from ...utils import (
+    OUTPUT_DIR,
+    current_timestamp,
+    deep_serialize,
+    from_env,
+    get_output_filename,
+    write_json,
+    write_summary,
+)
+from .assets import ConfluenceAsset
+from .client import ConfluenceClient, ConfluenceCredentials
+
+logger = logging.getLogger(__name__)
+
+
+def iterate_all_data(
+    client: ConfluenceClient,
+) -> Iterable[Tuple[ConfluenceAsset, Union[list, Iterator, dict]]]:
+    """Iterate over the extracted data from Confluence"""
+
+    logger.info("Extracting USERS from API")
+    users = list(deep_serialize(client.users()))
+    yield ConfluenceAsset.USERS, users
+    logger.info(f"Extracted {len(users)} users from API")
+
+    logger.info("Extracting PAGES from API")
+    pages = list(deep_serialize(client.pages()))
+    yield ConfluenceAsset.PAGES, pages
+    logger.info(f"Extracted {len(pages)} pages from API")
+
+
+def extract_all(**kwargs) -> None:
+    """
+    Extract data from Confluence API
+    Store the output files locally under the given output_directory
+    """
+
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+
+    credentials = ConfluenceCredentials(**kwargs)
+    client = ConfluenceClient(credentials=credentials)
+
+    ts = current_timestamp()
+
+    for key, data in iterate_all_data(client):
+        filename = get_output_filename(key.name.lower(), output_directory, ts)
+        write_json(filename, data)
+
+    es = current_timestamp()
+
+    write_summary(output_directory, ts, duration_second=es - ts)

castor_extractor/utils/deprecate.py

@@ -4,7 +4,7 @@ import warnings
 from typing import Tuple
 
 
-def deprecate_python(min_version_supported: Tuple[int]):
+def deprecate_python(min_version_supported: Tuple[int, ...]):
     """raises a warning if python version < min_version_supported"""
 
     python_version = (
@@ -22,5 +22,5 @@ def deprecate_python(min_version_supported: Tuple[int]):
     if python_version < min_version_supported:
         warnings.warn(message, DeprecationWarning)
 
-    # Since warnings are disabled by default, let's add a log as well
-    logging.warning(message)
+        # Since warnings are disabled by default, let's add a log as well
+        logging.warning(message)

castor_extractor/visualization/metabase/client/db/credentials.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field
+from pydantic import AliasChoices, Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 METABASE_DB_ENV_PREFIX = "CASTOR_METABASE_DB_"
@@ -18,9 +18,15 @@ class MetabaseDbCredentials(BaseSettings):
     host: str
     port: str
     database: str
-    schema_: str = Field(validation_alias=f"{METABASE_DB_ENV_PREFIX}SCHEMA")
+    schema_: str = Field(
+        validation_alias=AliasChoices(
+            f"{METABASE_DB_ENV_PREFIX}SCHEMA", "schema"
+        )
+    )
     user: str = Field(validation_alias=f"{METABASE_DB_ENV_PREFIX}USERNAME")
     password: str = Field(repr=False)
 
-    encryption_secret_key: Optional[str] = Field(repr=False)
-    require_ssl: Optional[str]
+    encryption_secret_key: Optional[str] = Field(repr=False, default=None)
+    require_ssl: Optional[bool] = Field(
+        validation_alias=f"{METABASE_DB_ENV_PREFIX}REQUIRE_SSL_KEY"
+    )

{castor_extractor-0.20.6.dist-info → castor_extractor-0.21.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.20.6
+Version: 0.21.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -37,7 +37,7 @@ Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=2,<3)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
-Requires-Dist: looker-sdk (>=24.0.0,<25.0.0) ; extra == "looker" or extra == "all"
+Requires-Dist: looker-sdk (>=24.16.0,<24.17.0) ; extra == "looker" or extra == "all"
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
 Requires-Dist: numpy (<1.25) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "bigquery" or extra == "databricks" or extra == "all")
 Requires-Dist: numpy (<2) ; extra == "bigquery" or extra == "databricks" or extra == "all"
@@ -208,6 +208,22 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.21.1 - 2024-10-23
+
+* Warning message to deprecate python < 3.9
+
+## 0.21.0 - 2024-10-23
+
+* Confluence: Added Confluence extractor
+
+## 0.20.8 - 2024-10-19
+
+* bump dependencies (minor and patches)
+
+## 0.20.7 - 2024-10-18
+
+* Metabase: fix `require_ssl` type in credentials
+
 ## 0.20.6 - 2024-10-15
 
 * Tableau: include `site_id` in **workbooks** to build url

{castor_extractor-0.20.6.dist-info → castor_extractor-0.21.1.dist-info}/RECORD

@@ -1,11 +1,12 @@
-CHANGELOG.md,sha256=Q6RN08q5JpWqwJjuR6cSm2k9vyeHbaNuufiVlS07vDo,14080
+CHANGELOG.md,sha256=U0folo6uva597R9IArkQzRau3UldJIdvq99aWOEWKkA,14355
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=uF6PXm9ocPITlKVSh9afTakHmpLx3TvawLf-CbMP3wM,3578
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-castor_extractor/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+castor_extractor/commands/__init__.py,sha256=qnCeLzbb8dlNJYr-P1ZMqHmNoXyaoojC8MkDUEZkfSk,116
 castor_extractor/commands/extract_bigquery.py,sha256=dU4OiYO1V0n32orvZnMh1_xtFKF_VxHNXcVsH3otY-g,1269
+castor_extractor/commands/extract_confluence.py,sha256=o4vjN7nxN2MQWyLgdgjbaO9Rd53f2icYst5tv_evwT4,750
 castor_extractor/commands/extract_databricks.py,sha256=SVKyoa-BBUQAM6HRHf1Wdg9-tpICic2yyvXQwHcNBhA,1264
 castor_extractor/commands/extract_domo.py,sha256=jvAawUsUTHrwCn_koK6StmQr4n_b5GyvJi6uu6WS0SM,1061
 castor_extractor/commands/extract_looker.py,sha256=cySLiolLCgrREJ9d0kMrJ7P8K3efHTBTzShalWVfI3A,1214
@@ -39,6 +40,14 @@ castor_extractor/file_checker/file_test_users_valid.csv,sha256=Ek3q7DjUS0neOu1LQ
 castor_extractor/file_checker/templates/__init__.py,sha256=StVLm4ZGyGVmPzarxEaDR_k08T3nUnyiv8N99sAz6AQ,60
 castor_extractor/file_checker/templates/generic_warehouse.py,sha256=3LREeiTsYsh5_wkuZost_frmMDp_K-jfiuod02vFMe8,2963
 castor_extractor/knowledge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+castor_extractor/knowledge/confluence/__init__.py,sha256=pRT615pMDlB7Ifs09erVn2EdpZHgkvX5selemWU3VPE,129
+castor_extractor/knowledge/confluence/assets.py,sha256=zv2G2LB8H0fKDbVJ4kHrAjbqehXI_K-wgd_ghSXGFvs,144
+castor_extractor/knowledge/confluence/client/__init__.py,sha256=ALAzo0JEhxFzH2FnIO6HmtkAGS2_bGY8KXXMcTGV3aE,84
+castor_extractor/knowledge/confluence/client/client.py,sha256=ZgAUlEGwpSN1QZAFGpU-7gob_wWI3wpBh1iTycxUZc4,1667
+castor_extractor/knowledge/confluence/client/credentials.py,sha256=2LUjnCOMwq2NBrkHty99TpWlgyOOsYjwC9NeekZMH84,422
+castor_extractor/knowledge/confluence/client/endpoints.py,sha256=b7PIvw9_W942L4zsEZa__KhZDTo4yt-CdIO0eas69TE,736
+castor_extractor/knowledge/confluence/client/pagination.py,sha256=ty4meiMEujDVSiQyOJTibd-ReYyDyGezdFuk7EAGtMA,862
+castor_extractor/knowledge/confluence/extract.py,sha256=0fnxqEQ1gLtNz0LANCvBHPCBN9hJSRMDWBgr0zNW45I,1545
 castor_extractor/knowledge/notion/__init__.py,sha256=ZDmh0eNSxHf1zVPm0aYlKPci-vzOXhAgdsWjS2hdjh4,117
 castor_extractor/knowledge/notion/assets.py,sha256=QHv1-pomt5UeN_prP2L6t_zJ-tDSqB8LgopkGAODYPQ,164
 castor_extractor/knowledge/notion/client/__init__.py,sha256=CDPorBCethuNTEtpjvHGcWnWeVfqkEq-IbakWjDKATw,76
@@ -94,7 +103,7 @@ castor_extractor/utils/dbt/assets.py,sha256=JY1nKEGySZ84wNoe7dnizwAYw2q0t8NVaIfq
 castor_extractor/utils/dbt/client.py,sha256=xBjbT-p99TXY850ooEAgjNp33yfGDwjWJRbXzeJoVaI,5538
 castor_extractor/utils/dbt/client_test.py,sha256=FO_vpnECE-hoK0rZHbqDv17oaJj3-uPhFEqTrMPzUf4,4533
 castor_extractor/utils/dbt/credentials.py,sha256=pGq7GqFQTw9TwN1DXSHC-0yJ2H6B_wMAbHyQTLqJVh0,543
-castor_extractor/utils/deprecate.py,sha256=_uzQiwHiz2yEqQeNMmzvVmBY46IgBhhEbGPhTrVjZU4,817
+castor_extractor/utils/deprecate.py,sha256=caXcCPStw9y97Bk_tu3pYVksfSj1x9ZyxnprNHJqHtA,830
 castor_extractor/utils/env.py,sha256=TqdtB50U8LE0993WhhEhpy89TJrHbjtIKjvg6KQ-5q0,596
 castor_extractor/utils/files.py,sha256=3C_u7P-kSZoOABVaKsuaf8lEhldRRxyxD27-K18_dEU,1545
 castor_extractor/utils/files_test.py,sha256=omRT3XSjaSAywYUoLh1SGWqYzl4UwBYKSYA9_7mXd_E,1542
@@ -167,7 +176,7 @@ castor_extractor/visualization/metabase/client/api/client_test.py,sha256=7Lb5yvr
 castor_extractor/visualization/metabase/client/api/credentials.py,sha256=SiHWyWWEDy_Ak4ne0hgXPZCSP6mEuHH26wDhsed_9v8,519
 castor_extractor/visualization/metabase/client/db/__init__.py,sha256=nawDhJ-JGlpM6VMzZZRjf066QXk9kWzZr6l9n6OHTZ0,76
 castor_extractor/visualization/metabase/client/db/client.py,sha256=s38mubBaun4LfYWXqk_fyObWTk9f69AorDrXyW4HNiw,3369
-castor_extractor/visualization/metabase/client/db/credentials.py,sha256=LrtfUd2c1jAAFzxu64suIwvZe2dJnZ8suv9vGTnXtrI,746
+castor_extractor/visualization/metabase/client/db/credentials.py,sha256=rhA6rM7QMTBSpF4BknYXz2VqujFcWAdx5TPR4kmdPy4,918
 castor_extractor/visualization/metabase/client/db/queries/.sqlfluff,sha256=sOQQOpAa9QMj9cBlulfmt-DZ_kQzMpzSAEnh10QGSB0,76
 castor_extractor/visualization/metabase/client/db/queries/base_url.sql,sha256=p2EL9kdt-hw_yh3aeCE91AXEB4RrYAbG2QrBBNqQjDE,79
 castor_extractor/visualization/metabase/client/db/queries/card.sql,sha256=U4-ZrZP5XS7Kssd3CFlpGrABbHAMK09Bh_yeT3z7Yk8,419
@@ -415,8 +424,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=j_d5-HMnzBouwGfywVZMRSSwbXzPvzDWlFCZmvxcoGQ,539
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.20.6.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.20.6.dist-info/METADATA,sha256=Vgkknt_--nGtd3FcSAsG5JxA6zvBVlc-lwSOqlXoeTE,21298
-castor_extractor-0.20.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.20.6.dist-info/entry_points.txt,sha256=IVGy_oM8VjzADMAxzmiNJTYYidTCsI98MpO_mkXjkqE,1573
-castor_extractor-0.20.6.dist-info/RECORD,,
+castor_extractor-0.21.1.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.21.1.dist-info/METADATA,sha256=28gV2BAN9hz_TdzFHMxezHikWiOLwTczn0BwVKGrCy0,21575
+castor_extractor-0.21.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.21.1.dist-info/entry_points.txt,sha256=7aVSxc-_2dicp28Ow-S4y0p4wGoTm9zGmVptMvfLdw8,1649
+castor_extractor-0.21.1.dist-info/RECORD,,

{castor_extractor-0.20.6.dist-info → castor_extractor-0.21.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any

{castor_extractor-0.20.6.dist-info → castor_extractor-0.21.1.dist-info}/entry_points.txt

@@ -1,5 +1,6 @@
 [console_scripts]
 castor-extract-bigquery=castor_extractor.commands.extract_bigquery:main
+castor-extract-confluence=castor_extractor.commands.extract_confluence:main
 castor-extract-databricks=castor_extractor.commands.extract_databricks:main
 castor-extract-domo=castor_extractor.commands.extract_domo:main
 castor-extract-looker=castor_extractor.commands.extract_looker:main