castor-extractor 0.24.35__py3-none-any.whl → 0.24.38__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 0.24.38 - 2025-08-07
+
+* Uploader: Support US and EU zones
+
+## 0.24.37 - 2025-08-06
+
+* Sigma: extract data models, dataset sources and workbook sources
+
+## 0.24.36 - 2025-08-04
+
+* Sigma:
+  * Refresh token before lineage extraction
+  * Disregard 403 errors during lineage extraction
+
 ## 0.24.35 - 2025-07-29
 
 * Coalesce - Fix pagination issue

castor_extractor/commands/upload.py

@@ -3,6 +3,7 @@ from argparse import ArgumentParser
 
 from castor_extractor.uploader import (  # type: ignore
     FileType,
+    Zone,
     upload_any,
 )
 from castor_extractor.utils import parse_filled_arguments  # type: ignore
@@ -40,6 +41,15 @@ def _args() -> ArgumentParser:
         ),
         choices=supported_file_type,
     )
+    supported_zones = [zone.value for zone in Zone]
+    parser.add_argument(
+        "-z",
+        "--zone",
+        help="geographic zone to upload, currently supported are {}, defaults to EU".format(
+            supported_zones,
+        ),
+        choices=supported_zones,
+    )
     return parser
 
 

castor_extractor/uploader/__init__.py

@@ -1,2 +1,3 @@
 from .constant import FileType
+from .enums import Zone
 from .upload import upload, upload_any, upload_manifest

castor_extractor/uploader/constant.py

@@ -1,9 +1,13 @@
 from enum import Enum
 
 from ..utils import RetryStrategy
+from .enums import Zone
 
 # url of the gcs proxy
-INGEST_URL = "https://ingest.castordoc.com"
+INGEST_URLS = {
+    Zone.EU: "https://ingest.castordoc.com",
+    Zone.US: "https://ingest.us.castordoc.com",
+}
 
 RETRY_BASE_MS = 10_000
 RETRY_JITTER_MS = 1_000

castor_extractor/uploader/enums.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class Zone(Enum):
+    """Geographic cluster location"""
+
+    EU = "EU"
+    US = "US"

castor_extractor/uploader/settings.py

@@ -4,6 +4,7 @@ from pydantic import UUID4, Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from .constant import FileType
+from .enums import Zone
 
 UPLOADER_ENV_PREFIX = "CASTOR_UPLOADER_"
 
@@ -22,3 +23,4 @@ class UploaderSettings(BaseSettings):
     file_type: FileType
     source_id: UUID4
     token: str = Field(repr=False)
+    zone: Optional[Zone] = Zone.EU

castor_extractor/uploader/upload.py

@@ -10,13 +10,14 @@ import requests
 
 from ..utils.retry import retry
 from .constant import (
-    INGEST_URL,
+    INGEST_URLS,
     PATH_TEMPLATES,
     RETRY_BASE_MS,
     RETRY_JITTER_MS,
     RETRY_STRATEGY,
     FileType,
 )
+from .enums import Zone
 from .env import get_blob_env
 from .settings import UploaderSettings
 from .utils import iter_files
@@ -33,6 +34,7 @@ def _path_and_url(
     source_id: UUID,
     file_type: FileType,
     file_path: str,
+    zone: Zone,
 ) -> tuple[str, str]:
     now = datetime.utcnow()
     timestamp = int(now.timestamp())
@@ -44,7 +46,7 @@ def _path_and_url(
         filename=filename,
     )
 
-    url = f"{INGEST_URL}/{path}"
+    url = f"{INGEST_URLS[zone]}/{path}"
 
     return path, url
 
@@ -61,13 +63,16 @@ def _upload(
     source_id: UUID,
     file_path: str,
     file_type: FileType,
+    zone: Optional[Zone] = Zone.EU,
 ) -> None:
     """
     Upload the given file to Google Cloud Storage (GCS)
     - Don't call GCS API directly
     - Call the ingestion proxy which handles authorisation and uploading
     """
-    path, url = _path_and_url(source_id, file_type, file_path)
+    if not zone:
+        zone = Zone.EU
+    path, url = _path_and_url(source_id, file_type, file_path, zone)
     headers = _headers(token)
     timeout, max_retries = get_blob_env()
 
@@ -97,6 +102,7 @@ def _upload(
 def upload_manifest(
     token: str,
     source_id: UUID,
+    zone: Optional[Zone],
     file_path: Optional[str] = None,
 ) -> None:
     """
@@ -106,13 +112,20 @@ def upload_manifest(
     """
     if not file_path:
         raise ValueError("file path is needed to upload a manifest")
-    _upload(token, source_id, file_path, FileType.DBT)
+    _upload(
+        token=token,
+        source_id=source_id,
+        file_path=file_path,
+        file_type=FileType.DBT,
+        zone=zone,
+    )
 
 
 def upload(
     token: str,
     source_id: UUID,
     file_type: FileType,
+    zone: Optional[Zone],
     file_path: Optional[str] = None,
     directory_path: Optional[str] = None,
 ) -> None:
@@ -133,7 +146,13 @@ def upload(
         raise ValueError(message)
 
     for file_ in files:
-        _upload(token, source_id, file_, file_type)
+        _upload(
+            token=token,
+            source_id=source_id,
+            file_path=file_,
+            file_type=file_type,
+            zone=zone,
+        )
 
 
 def upload_any(**kwargs) -> None:
@@ -156,6 +175,7 @@ def upload_any(**kwargs) -> None:
             token=settings.token,
             source_id=settings.source_id,
             file_path=settings.file_path,
+            zone=settings.zone,
         )
         return None
 
@@ -165,4 +185,5 @@ def upload_any(**kwargs) -> None:
         file_type=file_type,
         file_path=settings.file_path,
         directory_path=settings.directory_path,
+        zone=settings.zone,
     )

castor_extractor/uploader/upload_test.py

@@ -1,6 +1,7 @@
 from uuid import UUID
 
-from .constant import INGEST_URL, FileType
+from .constant import INGEST_URLS, FileType
+from .enums import Zone
 from .upload import _path_and_url
 
 
@@ -8,7 +9,8 @@ def test__path():
     source_id = UUID("399a8b22-3187-11ec-8d3d-0242ac130003")
     file_type = FileType.VIZ
     file_path = "filename"
+    zone = Zone.EU
 
-    path, url = _path_and_url(source_id, file_type, file_path)
+    path, url = _path_and_url(source_id, file_type, file_path, zone)
     assert path == f"visualization-{source_id}/{file_path}"
-    assert url == f"{INGEST_URL}/{path}"
+    assert url == f"{INGEST_URLS[Zone.EU]}/{path}"

castor_extractor/visualization/sigma/assets.py

@@ -4,10 +4,13 @@ from ...types import ExternalAsset
 class SigmaAsset(ExternalAsset):
     """Sigma assets"""
 
+    DATAMODELS = "datamodels"
     DATASETS = "datasets"
+    DATASET_SOURCES = "dataset_sources"
     ELEMENTS = "elements"
     FILES = "files"
     LINEAGES = "lineages"
     MEMBERS = "members"
     QUERIES = "queries"
     WORKBOOKS = "workbooks"
+    WORKBOOK_SOURCES = "workbook_sources"

castor_extractor/visualization/sigma/client/client.py

@@ -1,3 +1,4 @@
+import logging
 from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
@@ -24,6 +25,9 @@ from .pagination import (
     SIGMA_QUERIES_PAGINATION_LIMIT,
     SigmaPagination,
 )
+from .sources_transformer import SigmaSourcesTransformer
+
+logger = logging.getLogger(__name__)
 
 _CONTENT_TYPE = "application/x-www-form-urlencoded"
 
@@ -54,6 +58,13 @@ SIGMA_SAFE_MODE = RequestSafeMode(
     max_errors=_VOLUME_IGNORED,
     status_codes=_IGNORED_ERROR_CODES,
 )
+SIGMA_SAFE_MODE_LINEAGE = RequestSafeMode(
+    max_errors=_VOLUME_IGNORED,
+    status_codes=(
+        *_IGNORED_ERROR_CODES,
+        HTTPStatus.FORBIDDEN,
+    ),
+)
 _THREADS_LINEAGE = 10  # empirically found; hit the rate limit with 20 workers
 _RETRY_NUMBER = 1
 _RETRY_BASE_MS = 60_000
@@ -128,6 +139,12 @@ class SigmaClient(APIClient):
             params={"limit": limit},
         )
 
+    def _get_all_datamodels(self) -> Iterator[dict]:
+        request = self._get_paginated(
+            endpoint=SigmaEndpointFactory.datamodels()
+        )
+        yield from fetch_all_pages(request, SigmaPagination)
+
     def _get_all_datasets(self) -> Iterator[dict]:
         request = self._get_paginated(endpoint=SigmaEndpointFactory.datasets())
         yield from fetch_all_pages(request, SigmaPagination)
@@ -210,18 +227,35 @@ class SigmaClient(APIClient):
         return contexts
 
     def _get_all_lineages(self, elements: list[dict]) -> Iterator[dict]:
+        """
+        The safe mode is temporarily modified to include 403 errors.
+
+        Due to concurrency issues, we force a refresh of the token in hopes that
+        the lineage extraction takes less than the token expiration time of
+        1 hour.
+        """
+        safe_mode = self._safe_mode
+        self._safe_mode = SIGMA_SAFE_MODE_LINEAGE
+
         lineage_context = self._lineage_context(elements)
 
+        self._auth.refresh_token()
+
         with ThreadPoolExecutor(max_workers=_THREADS_LINEAGE) as executor:
             results = executor.map(self._get_lineage, lineage_context)
 
         for lineage in results:
+            if not lineage.lineage:
+                continue
+
             yield {
                 **lineage.lineage,
                 "workbook_id": lineage.context.workbook_id,
                 "element_id": lineage.context.element_id,
             }
 
+        self._safe_mode = safe_mode
+
     @staticmethod
     def _yield_deduplicated_queries(
         queries: Iterable[dict], workbook_id: str
@@ -251,18 +285,36 @@ class SigmaClient(APIClient):
 
             yield from self._yield_deduplicated_queries(queries, workbook_id)
 
+    def _get_all_dataset_sources(self, datasets: list[dict]) -> Iterator[dict]:
+        yield from SigmaSourcesTransformer(self).get_dataset_sources(datasets)
+
+    def _get_all_workbook_sources(
+        self, workbooks: list[dict]
+    ) -> Iterator[dict]:
+        yield from SigmaSourcesTransformer(self).get_workbook_sources(workbooks)
+
     def fetch(
         self,
         asset: SigmaAsset,
-        workbooks: Optional[list[dict]] = None,
+        datasets: Optional[list[dict]] = None,
         elements: Optional[list[dict]] = None,
+        workbooks: Optional[list[dict]] = None,
     ) -> Iterator[dict]:
         """Returns the needed metadata for the queried asset"""
-        if asset == SigmaAsset.DATASETS:
+        if asset == SigmaAsset.DATAMODELS:
+            yield from self._get_all_datamodels()
+
+        elif asset == SigmaAsset.DATASETS:
             yield from self._get_all_datasets()
 
+        elif asset == SigmaAsset.DATASET_SOURCES:
+            if datasets is None:
+                raise ValueError("Missing datasets to extract dataset sources")
+
+            yield from self._get_all_dataset_sources(datasets)
+
         elif asset == SigmaAsset.ELEMENTS:
-            if not workbooks:
+            if workbooks is None:
                 raise ValueError("Missing workbooks to extract elements")
 
             yield from self._get_all_elements(workbooks)
@@ -271,15 +323,16 @@ class SigmaClient(APIClient):
             yield from self._get_all_files()
 
         elif asset == SigmaAsset.LINEAGES:
-            if not elements:
+            if elements is None:
                 raise ValueError("Missing elements to extract lineage")
+
             yield from self._get_all_lineages(elements)
 
         elif asset == SigmaAsset.MEMBERS:
             yield from self._get_all_members()
 
         elif asset == SigmaAsset.QUERIES:
-            if not workbooks:
+            if workbooks is None:
                 raise ValueError("Missing workbooks to extract queries")
 
             yield from self._get_all_queries(workbooks)
@@ -287,5 +340,13 @@ class SigmaClient(APIClient):
         elif asset == SigmaAsset.WORKBOOKS:
             yield from self._get_all_workbooks()
 
+        elif asset == SigmaAsset.WORKBOOK_SOURCES:
+            if workbooks is None:
+                raise ValueError(
+                    "Missing workbooks to extract workbook sources"
+                )
+
+            yield from self._get_all_workbook_sources(workbooks)
+
         else:
             raise ValueError(f"This asset {asset} is unknown")

castor_extractor/visualization/sigma/client/endpoints.py

@@ -1,6 +1,7 @@
 class SigmaEndpointFactory:
     """Wrapper class around all endpoints we're using"""
 
+    DATAMODELS = "dataModels"
     DATASETS = "datasets"
     FILES = "files"
     MEMBERS = "members"
@@ -10,10 +11,22 @@ class SigmaEndpointFactory:
     def authentication(cls) -> str:
         return "v2/auth/token"
 
+    @classmethod
+    def connection_path(cls, inode_id: str) -> str:
+        return f"v2/connections/paths/{inode_id}"
+
+    @classmethod
+    def datamodels(cls) -> str:
+        return f"v2/{cls.DATAMODELS}"
+
     @classmethod
     def datasets(cls) -> str:
         return f"v2/{cls.DATASETS}"
 
+    @classmethod
+    def dataset_sources(cls, dataset_id: str) -> str:
+        return f"v2/{cls.DATASETS}/{dataset_id}/sources"
+
     @classmethod
     def elements(cls, workbook_id: str, page_id: str) -> str:
         return f"v2/{cls.WORKBOOKS}/{workbook_id}/pages/{page_id}/elements"
@@ -41,3 +54,7 @@ class SigmaEndpointFactory:
     @classmethod
     def workbooks(cls) -> str:
         return f"v2/{cls.WORKBOOKS}"
+
+    @classmethod
+    def workbook_sources(cls, workbook_id: str) -> str:
+        return f"v2/{cls.WORKBOOKS}/{workbook_id}/sources"

castor_extractor/visualization/sigma/client/sources_transformer.py

@@ -0,0 +1,94 @@
+import logging
+from typing import TYPE_CHECKING, Callable, Iterator
+
+from .endpoints import SigmaEndpointFactory
+
+if TYPE_CHECKING:
+    from .client import SigmaClient
+
+logger = logging.getLogger(__name__)
+
+
+class SigmaSourcesTransformer:
+    """Retrieves asset sources and enhances them with additional information."""
+
+    def __init__(self, api_client: "SigmaClient"):
+        self.api_client = api_client
+
+    def _map_table_id_to_connection_path(
+        self, all_sources: list
+    ) -> dict[str, dict]:
+        """Maps a table id to its connection and path information."""
+        logger.info("Mapping table ids to connection and path information")
+
+        unique_table_ids = {
+            source["inodeId"]
+            for asset_sources in all_sources
+            for source in asset_sources["sources"]
+            if source["type"] == "table"
+        }
+
+        return {
+            table_id: self.api_client._get(
+                endpoint=SigmaEndpointFactory.connection_path(table_id)
+            )
+            for table_id in unique_table_ids
+        }
+
+    @staticmethod
+    def _enhance_table_source(source: dict, table_to_path: dict) -> dict:
+        """
+        Combines a single table source with its connection and path information.
+        """
+        if source["type"] != "table":
+            return source
+
+        path_info = table_to_path.get(source["inodeId"], {})
+        source["connectionId"] = path_info.get("connectionId")
+        source["path"] = path_info.get("path")
+        return source
+
+    def _transform_sources(
+        self, all_sources: list, table_to_path: dict
+    ) -> Iterator[dict]:
+        """
+        Yields all sources, with table sources being enhanced with additional information.
+        """
+        logger.info("Merging sources with table information")
+
+        for asset_sources in all_sources:
+            enhanced_sources = [
+                self._enhance_table_source(source, table_to_path)
+                for source in asset_sources["sources"]
+            ]
+
+            yield {
+                "asset_id": asset_sources["asset_id"],
+                "sources": enhanced_sources,
+            }
+
+    def _get_all_sources(
+        self, endpoint: Callable[[str], str], asset_ids: set[str]
+    ) -> Iterator[dict]:
+        """Returns transformed sources for the given assets"""
+        all_sources = []
+
+        for asset_id in asset_ids:
+            sources = self.api_client._get(endpoint=endpoint(asset_id))
+            all_sources.append({"asset_id": asset_id, "sources": sources})
+
+        table_to_path = self._map_table_id_to_connection_path(all_sources)
+
+        yield from self._transform_sources(all_sources, table_to_path)
+
+    def get_dataset_sources(self, datasets: list[dict]) -> Iterator[dict]:
+        asset_ids = {dataset["datasetId"] for dataset in datasets}
+        yield from self._get_all_sources(
+            endpoint=SigmaEndpointFactory.dataset_sources, asset_ids=asset_ids
+        )
+
+    def get_workbook_sources(self, workbooks: list[dict]) -> Iterator[dict]:
+        asset_ids = {workbook["workbookId"] for workbook in workbooks}
+        yield from self._get_all_sources(
+            endpoint=SigmaEndpointFactory.workbook_sources, asset_ids=asset_ids
+        )

castor_extractor/visualization/sigma/client/sources_transformer_test.py

@@ -0,0 +1,101 @@
+from unittest.mock import Mock
+
+from .sources_transformer import SigmaSourcesTransformer
+
+_ALL_SOURCES = [
+    {
+        "asset_id": "asset1",
+        "sources": [
+            {"type": "dataset", "inodeId": "1234"},  # non-table source
+            {"type": "table", "inodeId": "table1"},
+            {"type": "table", "inodeId": "table2"},
+        ],
+    },
+    {
+        "asset_id": "asset2",
+        "sources": [
+            {"type": "table", "inodeId": "table1"},  # repeated source
+        ],
+    },
+]
+
+
+_TABLE_TO_PATH = {
+    "table1": {
+        "connectionId": "conn1",
+        "path": ["db", "schema", "table1"],
+    },
+    "table2": {
+        "connectionId": "conn2",
+        "path": ["db", "schema", "table2"],
+    },
+}
+
+
+def test__map_table_id_to_connection_path():
+    transformer = SigmaSourcesTransformer(api_client=Mock())
+
+    def mock_get(endpoint):
+        if "table1" in endpoint:
+            return _TABLE_TO_PATH["table1"]
+        elif "table2" in endpoint:
+            return _TABLE_TO_PATH["table2"]
+        else:
+            raise ValueError(f"Unexpected endpoint: {endpoint}")
+
+    transformer.api_client._get.side_effect = mock_get
+
+    result = transformer._map_table_id_to_connection_path(_ALL_SOURCES)
+
+    assert len(result) == 2
+    assert result["table1"] == {
+        "connectionId": "conn1",
+        "path": ["db", "schema", "table1"],
+    }
+    assert result["table2"] == {
+        "connectionId": "conn2",
+        "path": ["db", "schema", "table2"],
+    }
+    assert transformer.api_client._get.call_count == 2
+
+
+def test__transform_sources():
+    transformer = SigmaSourcesTransformer(api_client=Mock())
+
+    result = list(transformer._transform_sources(_ALL_SOURCES, _TABLE_TO_PATH))
+
+    assert len(result) == 2
+
+    asset_1_results = result[0]
+    assert len(asset_1_results["sources"]) == 3
+    actual_sources = sorted(
+        asset_1_results["sources"], key=lambda x: x["inodeId"]
+    )
+    expected_sources = [
+        {"type": "dataset", "inodeId": "1234"},
+        {
+            "type": "table",
+            "inodeId": "table1",
+            "connectionId": "conn1",
+            "path": ["db", "schema", "table1"],
+        },
+        {
+            "type": "table",
+            "inodeId": "table2",
+            "connectionId": "conn2",
+            "path": ["db", "schema", "table2"],
+        },
+    ]
+    expected_sources = sorted(expected_sources, key=lambda x: x["inodeId"])
+    assert actual_sources == expected_sources
+
+    asset_2_results = result[1]
+    assert asset_2_results["asset_id"] == "asset2"
+    assert asset_2_results["sources"] == [
+        {
+            "type": "table",
+            "inodeId": "table1",
+            "connectionId": "conn1",
+            "path": ["db", "schema", "table1"],
+        }
+    ]

castor_extractor/visualization/sigma/extract.py

@@ -22,14 +22,30 @@ def iterate_all_data(
 ) -> Iterable[tuple[SigmaAsset, Union[list, Iterator, dict]]]:
     """Iterate over the extracted data from Sigma"""
 
+    logger.info("Extracting DATA MODELS from API")
+    datamodels = client.fetch(SigmaAsset.DATAMODELS)
+    yield SigmaAsset.DATASETS, list(deep_serialize(datamodels))
+
     logger.info("Extracting DATASETS from API")
-    datasets = client.fetch(SigmaAsset.DATASETS)
+    datasets = list(client.fetch(SigmaAsset.DATASETS))
     yield SigmaAsset.DATASETS, list(deep_serialize(datasets))
 
+    logger.info("Extracting DATASET SOURCES from API")
+    dataset_sources = client.fetch(
+        SigmaAsset.DATASET_SOURCES, datasets=datasets
+    )
+    yield SigmaAsset.DATASET_SOURCES, list(deep_serialize(dataset_sources))
+
     logger.info("Extracting WORKBOOKS from API")
     workbooks = list(client.fetch(SigmaAsset.WORKBOOKS))
     yield SigmaAsset.WORKBOOKS, list(deep_serialize(workbooks))
 
+    logger.info("Extracting WORKBOOK SOURCES from API")
+    workbook_sources = client.fetch(
+        SigmaAsset.WORKBOOK_SOURCES, workbooks=workbooks
+    )
+    yield SigmaAsset.WORKBOOKS, list(deep_serialize(workbook_sources))
+
     logger.info("Extracting FILES from API")
     files = client.fetch(SigmaAsset.FILES)
     yield SigmaAsset.FILES, list(deep_serialize(files))

{castor_extractor-0.24.35.dist-info → castor_extractor-0.24.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.35
+Version: 0.24.38
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,20 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.38 - 2025-08-07
+
+* Uploader: Support US and EU zones
+
+## 0.24.37 - 2025-08-06
+
+* Sigma: extract data models, dataset sources and workbook sources
+
+## 0.24.36 - 2025-08-04
+
+* Sigma:
+  * Refresh token before lineage extraction
+  * Disregard 403 errors during lineage extraction
+
 ## 0.24.35 - 2025-07-29
 
 * Coalesce - Fix pagination issue

{castor_extractor-0.24.35.dist-info → castor_extractor-0.24.38.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=1S9O_c1LH8T4P78akRxlFS8Tv0i9Jgswy7V9zvd_UQw,18900
+CHANGELOG.md,sha256=cdsC0cY-q3t1K8a-kXhK3OY6y-yrF8uICKb8OqJ3SJo,19185
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -29,7 +29,7 @@ castor_extractor/commands/extract_strategy.py,sha256=Q-pUymatPrBFGXobhyUPzFph0-t
 castor_extractor/commands/extract_tableau.py,sha256=LNtI29LbVk1vp4RNrn89GmdW6R_7QBYunRmkowDhbco,1982
 castor_extractor/commands/extract_thoughtspot.py,sha256=caAYJlH-vK7u5IUB6OKXxcaWfLgc7d_XqnFDWK6YNS4,639
 castor_extractor/commands/file_check.py,sha256=TJx76Ymd0QCECmq35zRJMkPE8DJtSInB28MuSXWk8Ao,2644
-castor_extractor/commands/upload.py,sha256=rLXp7gQ8zb1kLbho4FT87q8eJd8Gvo_TkyIynAaQ-4s,1342
+castor_extractor/commands/upload.py,sha256=sqpEF_qqCNvT_niIrM6jPhzLaFVjtYwpc2iZw540F20,1633
 castor_extractor/file_checker/__init__.py,sha256=OSt6YLhUT42U_Cp3LCLHMVruwDkksL75Ij13X2UPnVk,119
 castor_extractor/file_checker/column.py,sha256=6bJhcW1snYwgHKkqlS0Ak7XLHZr4YBwO46JCIlnQNKg,3086
 castor_extractor/file_checker/column_test.py,sha256=1j8PxvmvmJgpd-mk30iMYOme32ovPSIn4yCXywFoXrg,1935
@@ -86,13 +86,14 @@ castor_extractor/transformation/dbt/client.py,sha256=BIue1DNAn2b7kHeiXBkGNosq8jZ
 castor_extractor/transformation/dbt/client_test.py,sha256=RLL7y_pLDv2QBM03qBht8yYEooeT_woRADHcb8vgBQ4,4535
 castor_extractor/transformation/dbt/credentials.py,sha256=pGq7GqFQTw9TwN1DXSHC-0yJ2H6B_wMAbHyQTLqJVh0,543
 castor_extractor/types.py,sha256=nHel2hv6NoHmdpOX_heEfO2-DnZPoYA2x0eJdbFvT0s,1276
-castor_extractor/uploader/__init__.py,sha256=A4bq_SrEtKAsl0r_D_duSTvL5WIQjVfsMy7tDx9IKg0,87
-castor_extractor/uploader/constant.py,sha256=yTigLHDlYwoRr6CpFIl7ReElFsQd4H-qkluMZJPWSx0,865
+castor_extractor/uploader/__init__.py,sha256=xe3QHmHb35TILEhr7__nI_0t0tDolpQuujUyd84YcjI,111
+castor_extractor/uploader/constant.py,sha256=ZmQtFx9nnR0GSLZ9k41upzV3ub4FJCUIyojIEVh-qIg,956
+castor_extractor/uploader/enums.py,sha256=s5KVeBZWRDbDu-qOnrJhTSkSqzh0gxv0W1Z4cUsXfb8,109
 castor_extractor/uploader/env.py,sha256=5KiWHV-WTHfF68T_vzI-ypKAxzy9b9fnz2y4T3lH6QY,871
 castor_extractor/uploader/env_test.py,sha256=ClCWWtwd2N-5ClIDUxVMeKkWfhhOTxpppsXUDmdjxSg,472
-castor_extractor/uploader/settings.py,sha256=3MvOX-UFRqrLZoiT7wYn9jUGro7NX4RCafYzrXrLQtA,590
-castor_extractor/uploader/upload.py,sha256=PSQfkO_7LSE0WBo9Tm_hlS2ONepKeB0cBFdJXySnues,4310
-castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
+castor_extractor/uploader/settings.py,sha256=sUZpg9eHemM99DMrBW8bnlMuoTmCmLCKq-D0OCuQbGA,649
+castor_extractor/uploader/upload.py,sha256=b2g9vWWjXWbt8Ms7brTc7OK_I7Z-1VSibNbppGoB2oQ,4764
+castor_extractor/uploader/upload_test.py,sha256=UgN7TnT9Chn6KVzRcAX0Tuvp7-tps3ugxGitlgb9TSY,462
 castor_extractor/uploader/utils.py,sha256=otAaySj5aeem6f0CTd0Te6ioJ6uP2J1p348j-SdIwDI,802
 castor_extractor/utils/__init__.py,sha256=z_BdKTUyuug3I5AzCuSGrAVskfLax4_olfORIjhZw_M,1691
 castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
@@ -269,14 +270,16 @@ castor_extractor/visualization/salesforce_reporting/client/rest.py,sha256=AqL1DT
 castor_extractor/visualization/salesforce_reporting/client/soql.py,sha256=ytZnX6zE-NoS_Kz12KghMcCM4ukPwhMj6U0rQZ_8Isk,1621
 castor_extractor/visualization/salesforce_reporting/extract.py,sha256=ScStilebLGf4HDTFqhVTQAvv_OrKxc8waycfBKdsVAc,1359
 castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13cMtnKNytiFRomwav27Q,114
-castor_extractor/visualization/sigma/assets.py,sha256=JZ1Cpxnml8P3mIJoTUM57hvylB18ErECQXaP5FF63O4,268
+castor_extractor/visualization/sigma/assets.py,sha256=uKGKDaeY1ejc7XGh4eFaNp2ygG7hgca132xsX4eCwKQ,380
 castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
-castor_extractor/visualization/sigma/client/client.py,sha256=ZE44k5klBVnc5lld3tpjuKGeSdFmlJ0wr5DOB4pEfco,9446
+castor_extractor/visualization/sigma/client/client.py,sha256=VU0BHlug3tCpGA1je0PjEy4hU4TKhCH9UUGi8LRmNy8,11422
 castor_extractor/visualization/sigma/client/client_test.py,sha256=ae0ZOvKutCm44jnrJ-0_A5Y6ZGyDkMf9Ml3eEP8dNkY,581
 castor_extractor/visualization/sigma/client/credentials.py,sha256=XddAuQSmCKpxJ70TQgRnOj0vMPYVtiStk_lMMQ1AiNM,693
-castor_extractor/visualization/sigma/client/endpoints.py,sha256=DBFphbgoH78_MZUGM_bKBAq28Nl7LWSZ6VRsbxrxtDg,1162
+castor_extractor/visualization/sigma/client/endpoints.py,sha256=i7KTKnl2Os6752CdtJl0vPSC_Z6JxmacodV_saOnce0,1662
 castor_extractor/visualization/sigma/client/pagination.py,sha256=2bFA7GiBUUasFtHJKA90516d283p7Pg50-4zw6Fwt8I,726
-castor_extractor/visualization/sigma/extract.py,sha256=XIT1qsj6g6dgBWP8HPfj_medZexu48EaY9tUwi14gzM,2298
+castor_extractor/visualization/sigma/client/sources_transformer.py,sha256=mRupzxjtjDqELIouHF0egBkgslDmn5Y4uqO_sbUGCNs,3244
+castor_extractor/visualization/sigma/client/sources_transformer_test.py,sha256=06yUHXyv65amXLKXhix6K3kkVc1kpBqSjIYcxbyMI4Y,2766
+castor_extractor/visualization/sigma/extract.py,sha256=poTh70Xm2D6BwbdGApLkjXy6-t4iZnOoMB5DPfaTLEI,2929
 castor_extractor/visualization/strategy/__init__.py,sha256=HOMv4JxqF5ZmViWi-pDE-PSXJRLTdXal_jtpHG_rlR8,123
 castor_extractor/visualization/strategy/assets.py,sha256=yFXF_dX01patC0HQ1eU7Jo_4DZ4m6IJEg0uCB71tMoI,480
 castor_extractor/visualization/strategy/client/__init__.py,sha256=XWP0yF5j6JefDJkDfX-RSJn3HF2ceQ0Yx1PLCfB3BBo,80
@@ -428,8 +431,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=4RgeSkHDWTWRyU2iLx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=7sW8cK3JzxPt6faTJ7e4lk9tE4fo_AeCymI-LqsSols,1276
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.35.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.35.dist-info/METADATA,sha256=-vrfKzS5B3r2qL7tjFjFBR-AizzuVIexEVJHCci7Z5s,26353
-castor_extractor-0.24.35.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.35.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
-castor_extractor-0.24.35.dist-info/RECORD,,
+castor_extractor-0.24.38.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.38.dist-info/METADATA,sha256=iCWUVbgDFS721szJ8kUGMA58Va3Roq3WmyGinZgnHMw,26638
+castor_extractor-0.24.38.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.38.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
+castor_extractor-0.24.38.dist-info/RECORD,,