castor-extractor 0.24.35__py3-none-any.whl → 0.24.36__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.24.36 - 2025-08-04
+
+* Sigma:
+  * Refresh token before lineage extraction
+  * Disregard 403 errors during lineage extraction
+
 ## 0.24.35 - 2025-07-29
 
 * Coalesce - Fix pagination issue

castor_extractor/visualization/sigma/client/client.py

@@ -54,6 +54,13 @@ SIGMA_SAFE_MODE = RequestSafeMode(
     max_errors=_VOLUME_IGNORED,
     status_codes=_IGNORED_ERROR_CODES,
 )
+SIGMA_SAFE_MODE_LINEAGE = RequestSafeMode(
+    max_errors=_VOLUME_IGNORED,
+    status_codes=(
+        *_IGNORED_ERROR_CODES,
+        HTTPStatus.FORBIDDEN,
+    ),
+)
 _THREADS_LINEAGE = 10  # empirically found; hit the rate limit with 20 workers
 _RETRY_NUMBER = 1
 _RETRY_BASE_MS = 60_000
@@ -210,18 +217,35 @@ class SigmaClient(APIClient):
         return contexts
 
     def _get_all_lineages(self, elements: list[dict]) -> Iterator[dict]:
+        """
+        The safe mode is temporarily modified to include 403 errors.
+
+        Due to concurrency issues, we force a refresh of the token in hopes that
+        the lineage extraction takes less than the token expiration time of
+        1 hour.
+        """
+        safe_mode = self._safe_mode
+        self._safe_mode = SIGMA_SAFE_MODE_LINEAGE
+
         lineage_context = self._lineage_context(elements)
 
+        self._auth.refresh_token()
+
         with ThreadPoolExecutor(max_workers=_THREADS_LINEAGE) as executor:
             results = executor.map(self._get_lineage, lineage_context)
 
         for lineage in results:
+            if not lineage.lineage:
+                continue
+
             yield {
                 **lineage.lineage,
                 "workbook_id": lineage.context.workbook_id,
                 "element_id": lineage.context.element_id,
             }
 
+        self._safe_mode = safe_mode
+
     @staticmethod
     def _yield_deduplicated_queries(
         queries: Iterable[dict], workbook_id: str

{castor_extractor-0.24.35.dist-info → castor_extractor-0.24.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.35
+Version: 0.24.36
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,12 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.36 - 2025-08-04
+
+* Sigma:
+  * Refresh token before lineage extraction
+  * Disregard 403 errors during lineage extraction
+
 ## 0.24.35 - 2025-07-29
 
 * Coalesce - Fix pagination issue

{castor_extractor-0.24.35.dist-info → castor_extractor-0.24.36.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=1S9O_c1LH8T4P78akRxlFS8Tv0i9Jgswy7V9zvd_UQw,18900
+CHANGELOG.md,sha256=HAHFgRYnv-pbsKwbHrRCrWoLpsqr8mg7Fp7tDsBsN9E,19030
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -271,7 +271,7 @@ castor_extractor/visualization/salesforce_reporting/extract.py,sha256=ScStilebLG
 castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13cMtnKNytiFRomwav27Q,114
 castor_extractor/visualization/sigma/assets.py,sha256=JZ1Cpxnml8P3mIJoTUM57hvylB18ErECQXaP5FF63O4,268
 castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
-castor_extractor/visualization/sigma/client/client.py,sha256=ZE44k5klBVnc5lld3tpjuKGeSdFmlJ0wr5DOB4pEfco,9446
+castor_extractor/visualization/sigma/client/client.py,sha256=ifCxhZ8-p9u7MnJRE8EYF_YP_G3REr_PELTSrtHiZwk,10099
 castor_extractor/visualization/sigma/client/client_test.py,sha256=ae0ZOvKutCm44jnrJ-0_A5Y6ZGyDkMf9Ml3eEP8dNkY,581
 castor_extractor/visualization/sigma/client/credentials.py,sha256=XddAuQSmCKpxJ70TQgRnOj0vMPYVtiStk_lMMQ1AiNM,693
 castor_extractor/visualization/sigma/client/endpoints.py,sha256=DBFphbgoH78_MZUGM_bKBAq28Nl7LWSZ6VRsbxrxtDg,1162
@@ -428,8 +428,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=4RgeSkHDWTWRyU2iLx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=7sW8cK3JzxPt6faTJ7e4lk9tE4fo_AeCymI-LqsSols,1276
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.35.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.35.dist-info/METADATA,sha256=-vrfKzS5B3r2qL7tjFjFBR-AizzuVIexEVJHCci7Z5s,26353
-castor_extractor-0.24.35.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.35.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
-castor_extractor-0.24.35.dist-info/RECORD,,
+castor_extractor-0.24.36.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.36.dist-info/METADATA,sha256=m14Hk_AYJo9_bZE7IOb6U_LdhG8JfXnVqisiJHjgMS4,26483
+castor_extractor-0.24.36.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.36.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
+castor_extractor-0.24.36.dist-info/RECORD,,