castor-extractor 0.20.7__tar.gz → 0.21.1__tar.gz

{castor_extractor-0.20.7 → castor_extractor-0.21.1}/CHANGELOG.md

@@ -1,6 +1,18 @@
 
 # Changelog
 
+## 0.21.1 - 2024-10-23
+
+* Warning message to deprecate python < 3.9
+
+## 0.21.0 - 2024-10-23
+
+* Confluence: Added Confluence extractor
+
+## 0.20.8 - 2024-10-19
+
+* bump dependencies (minor and patches)
+
 ## 0.20.7 - 2024-10-18
 
 * Metabase: fix `require_ssl` type in credentials

{castor_extractor-0.20.7 → castor_extractor-0.21.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.20.7
+Version: 0.21.1
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -37,7 +37,7 @@ Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
 Requires-Dist: google-cloud-storage (>=2,<3)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
-Requires-Dist: looker-sdk (>=24.0.0,<25.0.0) ; extra == "looker" or extra == "all"
+Requires-Dist: looker-sdk (>=24.16.0,<24.17.0) ; extra == "looker" or extra == "all"
 Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
 Requires-Dist: numpy (<1.25) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "bigquery" or extra == "databricks" or extra == "all")
 Requires-Dist: numpy (<2) ; extra == "bigquery" or extra == "databricks" or extra == "all"
@@ -208,6 +208,18 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.21.1 - 2024-10-23
+
+* Warning message to deprecate python < 3.9
+
+## 0.21.0 - 2024-10-23
+
+* Confluence: Added Confluence extractor
+
+## 0.20.8 - 2024-10-19
+
+* bump dependencies (minor and patches)
+
 ## 0.20.7 - 2024-10-18
 
 * Metabase: fix `require_ssl` type in credentials

castor_extractor-0.21.1/castor_extractor/commands/__init__.py

@@ -0,0 +1,3 @@
+from castor_extractor.utils import deprecate_python  # type: ignore
+
+deprecate_python(min_version_supported=(3, 9))

castor_extractor-0.21.1/castor_extractor/commands/extract_confluence.py

@@ -0,0 +1,19 @@
+import logging
+from argparse import ArgumentParser
+
+from castor_extractor.knowledge import confluence  # type: ignore
+from castor_extractor.utils import parse_filled_arguments  # type: ignore
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+
+
+def main():
+    parser = ArgumentParser()
+
+    parser.add_argument("-a", "--account_id", help="Confluence account id")
+    parser.add_argument("-b", "--base_url", help="Confluence account base url")
+    parser.add_argument("-o", "--output", help="Directory to write to")
+    parser.add_argument("-p", "--password", help="Confluence password")
+    parser.add_argument("-u", "--username", help="Confluence username")
+
+    confluence.extract_all(**parse_filled_arguments(parser))

castor_extractor-0.21.1/castor_extractor/knowledge/confluence/__init__.py

@@ -0,0 +1,3 @@
+from .assets import ConfluenceAsset
+from .client import ConfluenceClient, ConfluenceCredentials
+from .extract import extract_all

castor_extractor-0.21.1/castor_extractor/knowledge/confluence/assets.py

@@ -0,0 +1,8 @@
+from ...types import ExternalAsset
+
+
+class ConfluenceAsset(ExternalAsset):
+    """Confluence assets"""
+
+    PAGES = "pages"
+    USERS = "users"

castor_extractor-0.21.1/castor_extractor/knowledge/confluence/client/__init__.py

@@ -0,0 +1,2 @@
+from .client import ConfluenceClient
+from .credentials import ConfluenceCredentials

castor_extractor-0.21.1/castor_extractor/knowledge/confluence/client/client.py

@@ -0,0 +1,60 @@
+from functools import partial
+from http import HTTPStatus
+from typing import Optional
+
+from ....utils import (
+    APIClient,
+    BasicAuth,
+    RequestSafeMode,
+    fetch_all_pages,
+)
+from .credentials import ConfluenceCredentials
+from .endpoints import ConfluenceEndpointFactory
+from .pagination import ConfluencePagination
+
+_HEADERS = {
+    "Accept": "application/json",
+    "Content-Type": "application/json",
+}
+
+_MAX_ERROR_IGNORED_COUNT = 10
+_IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY,)
+_SAFE_MODE = RequestSafeMode(
+    max_errors=_MAX_ERROR_IGNORED_COUNT,
+    status_codes=_IGNORED_ERROR_CODES,
+)
+
+
+class ConfluenceClient(APIClient):
+    def __init__(
+        self,
+        credentials: ConfluenceCredentials,
+        safe_mode: Optional[RequestSafeMode] = None,
+    ):
+        self.account_id = credentials.account_id
+        auth = BasicAuth(
+            username=credentials.username, password=credentials.password
+        )
+        super().__init__(
+            auth=auth,
+            host=credentials.base_url,
+            headers=_HEADERS,
+            safe_mode=safe_mode or _SAFE_MODE,
+        )
+
+    def pages(self):
+        request = partial(
+            self._get,
+            endpoint=ConfluenceEndpointFactory.pages(),
+            params={"body-format": "atlas_doc_format"},
+        )
+        yield from fetch_all_pages(request, ConfluencePagination)
+
+    def users(self):
+        request_body = {"accountIds": [self.account_id]}
+        request = partial(
+            self._post,
+            endpoint=ConfluenceEndpointFactory.users(),
+            data=request_body,
+        )
+        yield from fetch_all_pages(request, ConfluencePagination)

castor_extractor-0.21.1/castor_extractor/knowledge/confluence/client/credentials.py

@@ -0,0 +1,18 @@
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+CASTOR_ENV_PREFIX = "CASTOR_CONFLUENCE_"
+
+
+class ConfluenceCredentials(BaseSettings):
+    """Class to handle Confluence rest API permissions"""
+
+    model_config = SettingsConfigDict(
+        env_prefix=CASTOR_ENV_PREFIX,
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    account_id: str
+    base_url: str
+    password: str
+    username: str

castor_extractor-0.21.1/castor_extractor/knowledge/confluence/client/endpoints.py

@@ -0,0 +1,25 @@
+class ConfluenceEndpointFactory:
+    """
+    Confluence rest api v2 endpoint factory.
+    https://developer.atlassian.com/cloud/confluence/rest/v2/intro/#about
+    """
+
+    API = "wiki/api/v2/"
+    PAGES = "pages"
+    USERS = "users-bulk"
+
+    @classmethod
+    def pages(cls) -> str:
+        """
+        Endpoint to fetch all pages.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-pages-get
+        """
+        return f"{cls.API}{cls.PAGES}"
+
+    @classmethod
+    def users(cls) -> str:
+        """
+        Endpoint to fetch all user.
+        More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-user/#api-users-bulk-post
+        """
+        return f"{cls.API}{cls.USERS}"

castor_extractor-0.21.1/castor_extractor/knowledge/confluence/client/pagination.py

@@ -0,0 +1,32 @@
+from typing import Optional, Union
+
+from pydantic import AliasPath, Field
+
+from ....utils import (
+    FetchNextPageBy,
+    PaginationModel,
+)
+
+
+class ConfluencePagination(PaginationModel):
+    """Class to handle paginated results for confluence"""
+
+    fetch_by: FetchNextPageBy = FetchNextPageBy.URL
+
+    results: list = Field(default_factory=list)
+    next_url: Optional[str] = Field(
+        validation_alias=AliasPath("_links", "next"),
+        default=None,
+    )
+
+    def is_last(self) -> bool:
+        """Stopping condition for the pagination"""
+        return self.next_url is None
+
+    def next_page_payload(self) -> Union[str, None]:
+        """Payload enabling to generate the request for the next page"""
+        return self.next_url
+
+    def page_results(self) -> list:
+        """List of results of the current page"""
+        return self.results

castor_extractor-0.21.1/castor_extractor/knowledge/confluence/extract.py

@@ -0,0 +1,54 @@
+import logging
+from typing import Iterable, Iterator, Tuple, Union
+
+from ...utils import (
+    OUTPUT_DIR,
+    current_timestamp,
+    deep_serialize,
+    from_env,
+    get_output_filename,
+    write_json,
+    write_summary,
+)
+from .assets import ConfluenceAsset
+from .client import ConfluenceClient, ConfluenceCredentials
+
+logger = logging.getLogger(__name__)
+
+
+def iterate_all_data(
+    client: ConfluenceClient,
+) -> Iterable[Tuple[ConfluenceAsset, Union[list, Iterator, dict]]]:
+    """Iterate over the extracted data from Confluence"""
+
+    logger.info("Extracting USERS from API")
+    users = list(deep_serialize(client.users()))
+    yield ConfluenceAsset.USERS, users
+    logger.info(f"Extracted {len(users)} users from API")
+
+    logger.info("Extracting PAGES from API")
+    pages = list(deep_serialize(client.pages()))
+    yield ConfluenceAsset.PAGES, pages
+    logger.info(f"Extracted {len(pages)} pages from API")
+
+
+def extract_all(**kwargs) -> None:
+    """
+    Extract data from Confluence API
+    Store the output files locally under the given output_directory
+    """
+
+    output_directory = kwargs.get("output") or from_env(OUTPUT_DIR)
+
+    credentials = ConfluenceCredentials(**kwargs)
+    client = ConfluenceClient(credentials=credentials)
+
+    ts = current_timestamp()
+
+    for key, data in iterate_all_data(client):
+        filename = get_output_filename(key.name.lower(), output_directory, ts)
+        write_json(filename, data)
+
+    es = current_timestamp()
+
+    write_summary(output_directory, ts, duration_second=es - ts)

{castor_extractor-0.20.7 → castor_extractor-0.21.1}/castor_extractor/utils/deprecate.py

@@ -4,7 +4,7 @@ import warnings
 from typing import Tuple
 
 
-def deprecate_python(min_version_supported: Tuple[int]):
+def deprecate_python(min_version_supported: Tuple[int, ...]):
     """raises a warning if python version < min_version_supported"""
 
     python_version = (
@@ -22,5 +22,5 @@ def deprecate_python(min_version_supported: Tuple[int]):
     if python_version < min_version_supported:
         warnings.warn(message, DeprecationWarning)
 
-    # Since warnings are disabled by default, let's add a log as well
-    logging.warning(message)
+        # Since warnings are disabled by default, let's add a log as well
+        logging.warning(message)

{castor_extractor-0.20.7 → castor_extractor-0.21.1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.20.7"
+version = "0.21.1"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -34,7 +34,7 @@ google-cloud-core = ">=2.1.0, <3.0.0"
 google-cloud-storage = "^2"
 google-resumable-media = "^2.0.3"
 googleapis-common-protos = "^1.53.0"
-looker-sdk = { version = "^24.0.0", optional = true }
+looker-sdk = { version = "~24.16.0", optional = true } # https://github.com/looker-open-source/sdk-codegen/issues/1518 -> lock until t
 msal = { version = "^1.20.0", optional = true }
 numpy = [ # Subdependency of pyarrow. As numpy remove 3.8 in 1.25 and add 3.12 in 1.26. And poetry doesn't manage it well.
   { version = "<1.25", python = "~3.8", optional = true },
@@ -115,6 +115,7 @@ deptry = "^0.20"
 
 [tool.poetry.scripts]
 castor-extract-bigquery = "castor_extractor.commands.extract_bigquery:main"
+castor-extract-confluence = "castor_extractor.commands.extract_confluence:main"
 castor-extract-databricks = "castor_extractor.commands.extract_databricks:main"
 castor-extract-domo = "castor_extractor.commands.extract_domo:main"
 castor-extract-looker = "castor_extractor.commands.extract_looker:main"