castor-extractor 0.20.4__py3-none-any.whl → 0.20.5__py3-none-any.whl

CHANGELOG.md

@@ -1,6 +1,10 @@
 
 # Changelog
 
+## 0.20.5 - 2024-10-09
+
+* Redshift: enable extraction from a Redshift Serverless instance
+
 ## 0.20.4 - 2024-10-09
 
 * Salesforce warehouse: `Labels` instead of `api_names` for columns

castor_extractor/commands/extract_redshift.py

@@ -23,6 +23,11 @@ def main():
         action="store_true",
         help="Skips files already extracted instead of replacing them",
     )
+    parser.add_argument(
+        "--serverless",
+        action="store_true",
+        help="Enables extraction for Redshift Serverless",
+    )
     parser.set_defaults(skip_existing=False)
 
     args = parser.parse_args()
@@ -34,5 +39,6 @@ def main():
         user=args.user,
         password=args.password,
         output_directory=args.output,
+        serverless=args.serverless,
         skip_existing=args.skip_existing,
     )

castor_extractor/warehouse/redshift/extract.py

@@ -34,6 +34,7 @@ REDSHIFT_PASSWORD = "CASTOR_REDSHIFT_PASSWORD"  # noqa: S105
 REDSHIFT_HOST = "CASTOR_REDSHIFT_HOST"
 REDSHIFT_PORT = "CASTOR_REDSHIFT_PORT"
 REDSHIFT_DATABASE = "CASTOR_REDSHIFT_DATABASE"
+REDSHIFT_SERVERLESS = "CASTOR_REDSHIFT_SERVERLESS"
 
 
 def _credentials(params: dict) -> dict:
@@ -48,6 +49,14 @@ def _credentials(params: dict) -> dict:
     }
 
 
+def _query_builder(params: dict) -> RedshiftQueryBuilder:
+    env_parameter = from_env(REDSHIFT_SERVERLESS, allow_missing=True)
+    from_env_ = str(env_parameter).lower() == "true"
+    from_params_ = params.get("serverless", False)
+    is_serverless = from_params_ or from_env_
+    return RedshiftQueryBuilder(is_serverless=is_serverless)
+
+
 def extract_all(**kwargs) -> None:
     """
     Extract all assets from Redshift and store the results in CSV files
@@ -56,7 +65,7 @@ def extract_all(**kwargs) -> None:
 
     client = RedshiftClient(credentials=_credentials(kwargs))
 
-    query_builder = RedshiftQueryBuilder()
+    query_builder = _query_builder(kwargs)
 
     storage = LocalStorage(directory=output_directory)
 

castor_extractor/warehouse/redshift/extract_test.py

@@ -0,0 +1,26 @@
+import pytest
+
+from .extract import (
+    REDSHIFT_SERVERLESS,
+    _query_builder,
+)
+
+
+@pytest.mark.parametrize(
+    "serverless_param,env_param,expected",
+    [
+        (True, "False", True),
+        (False, "True", True),
+        (None, "TRUE", True),
+        (None, "TrUe", True),
+        (None, "FAlSE", False),
+        (None, "False", False),
+        (None, None, False),
+        (True, None, True),
+    ],
+)
+def test__query_builder(serverless_param, env_param, expected, monkeypatch):
+    params = {"serverless": serverless_param}
+    monkeypatch.setenv(REDSHIFT_SERVERLESS, env_param)
+
+    assert _query_builder(params).is_serverless == expected

castor_extractor/warehouse/redshift/queries/query_serverless.sql

@@ -0,0 +1,69 @@
+WITH parameters AS (
+    SELECT
+        :day AS day_start,
+        :hour_min AS hour_min,
+        :hour_max AS hour_max
+),
+
+queries_deduplicated AS (
+    SELECT DISTINCT q.query_id
+    FROM SYS_QUERY_HISTORY AS q
+        CROSS JOIN parameters AS p
+    WHERE TRUE
+        AND DATE(q.start_time) = p.day_start
+        AND EXTRACT('hour' FROM q.start_time) BETWEEN p.hour_min AND p.hour_max
+),
+
+query AS (
+    SELECT
+        q.query_id,
+        qt.text,
+        qt.sequence,
+        COUNT(*) OVER(PARTITION BY q.query_id) AS sequence_count
+    FROM queries_deduplicated AS q
+        INNER JOIN SYS_QUERY_TEXT AS qt ON q.query_id = qt.query_id
+),
+
+raw_query_text AS
+(
+    SELECT
+        q.query_id,
+        LISTAGG(q.text, '') WITHIN GROUP (ORDER BY q.sequence) AS agg_text
+    FROM query AS q
+    WHERE TRUE
+    -- LISTAGG raises an error when total length >= 65535
+    -- each query text contains 4000 char max
+        AND q.sequence_count < (65535 / 4000)
+    GROUP BY q.query_id
+),
+
+query_text AS (
+    SELECT
+        query_id,
+        CASE
+            WHEN agg_text ILIKE 'INSERT INTO%%'
+                THEN REGEXP_REPLACE(agg_text, 'VALUES (.*)', 'DEFAULT VALUES')
+            ELSE agg_text
+        END AS agg_text
+    FROM raw_query_text
+)
+SELECT
+    q.query_id::VARCHAR(256) AS query_id,
+    qt.agg_text::VARCHAR(60000) AS query_text,
+    q.database_name AS database_id,
+    q.database_name AS database_name,
+    q.session_id AS process_id,
+    0 as aborted,
+    q.start_time AS start_time,
+    q.end_time AS end_time,
+    q.user_id AS user_id,
+    q.query_label,
+    u.usename AS user_name
+FROM SYS_QUERY_HISTORY AS q
+    JOIN query_text AS qt ON q.query_id = qt.query_id
+    JOIN pg_catalog.pg_user AS u ON u.usesysid = q.user_id
+    CROSS JOIN parameters AS p
+WHERE TRUE
+    AND DATE(q.start_time) = p.day_start
+    AND EXTRACT('hour' FROM q.start_time) BETWEEN p.hour_min AND p.hour_max
+    AND q.status = 'success'

castor_extractor/warehouse/redshift/query.py

@@ -15,10 +15,21 @@ class RedshiftQueryBuilder(AbstractQueryBuilder):
 
     def __init__(
         self,
+        is_serverless: bool = False,
         time_filter: Optional[TimeFilter] = None,
     ):
         super().__init__(time_filter=time_filter)
+        self.is_serverless = is_serverless
+
+    def build_query_serverless(self) -> ExtractionQuery:
+        """To get the query history in Redshift Serverless, we cannot use STL tables."""
+        statement = self._load_from_file("query_serverless.sql")
+        params = self._time_filter.to_dict()
+        return ExtractionQuery(statement, params)
 
     def build(self, asset: WarehouseAsset) -> List[ExtractionQuery]:
-        query = self.build_default(asset)
+        if asset == WarehouseAsset.QUERY and self.is_serverless:
+            query = self.build_query_serverless()
+        else:
+            query = self.build_default(asset)
         return [query]

{castor_extractor-0.20.4.dist-info → castor_extractor-0.20.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.20.4
+Version: 0.20.5
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -208,6 +208,10 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.20.5 - 2024-10-09
+
+* Redshift: enable extraction from a Redshift Serverless instance
+
 ## 0.20.4 - 2024-10-09
 
 * Salesforce warehouse: `Labels` instead of `api_names` for columns

{castor_extractor-0.20.4.dist-info → castor_extractor-0.20.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=CzVaQbFAS2hlZE2ak7DTYHWBNjMaC59e8UK7Q9p10tw,13905
+CHANGELOG.md,sha256=gqprCxUMpvdGZdxp3AxdsRKA3JK-Q9NM72m839G5CeQ,13996
 Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -17,7 +17,7 @@ castor_extractor/commands/extract_notion.py,sha256=uaxcF3_bT7D_-JxnIW0F7VVDphI_Z
 castor_extractor/commands/extract_postgres.py,sha256=pX0RnCPi4nw6QQ6wiAuZ_Xt3ZbDuMUG9aQKuqFgJtAU,1154
 castor_extractor/commands/extract_powerbi.py,sha256=f0G5w61KXExJ6Sw39_mJIwqQNpLorE5-LKmZXlUqvKI,783
 castor_extractor/commands/extract_qlik.py,sha256=VBe_xFKh_nR0QSFFIncAaC8yDqBeMa6VunBAga7AeGg,891
-castor_extractor/commands/extract_redshift.py,sha256=bdLp7d7ImZoKCkWc3f3NXF1imIzMVT43_KPI-x4UVac,1155
+castor_extractor/commands/extract_redshift.py,sha256=zRBg2D_ft4GLdPSdmetRcgQVAA80DXtdRSYsQhAWIik,1334
 castor_extractor/commands/extract_salesforce.py,sha256=3j3YTmMkPAwocR-B1ozJQai0UIZPtpmAyWj-hHvdWn4,1226
 castor_extractor/commands/extract_salesforce_reporting.py,sha256=FdANTNiLkIPdm80XMYxWReHjdycLsIa61pyeCD-sUDk,962
 castor_extractor/commands/extract_sigma.py,sha256=sxewHcZ1Doq35V2qnpX_zCKKXkrb1_9bYjUMg7BOW-k,643
@@ -362,18 +362,20 @@ castor_extractor/warehouse/postgres/query.py,sha256=5QmI79BP_EjqxeABNg56rxuM9Xuu
 castor_extractor/warehouse/redshift/__init__.py,sha256=CC82SejYDlwYhZhhn40ln-oTsRx7AJ1Km61cxPkymjE,125
 castor_extractor/warehouse/redshift/client.py,sha256=My7003HGBhTpS6X5NgYcKwntR7h45scLaGr-LSY0tIc,2172
 castor_extractor/warehouse/redshift/client_test.py,sha256=74lZfna71qs80EKAuitJ8_ZjAGtpYHf5tChySinVPoQ,1023
-castor_extractor/warehouse/redshift/extract.py,sha256=XnAnBSapzXYUWVQKMhVaLlloA-uXEqseOhSt9flVsdw,2331
+castor_extractor/warehouse/redshift/extract.py,sha256=pblUQ2XafVTpyHrDLrmwFKy55mUNF03dZvgPweihYUc,2723
+castor_extractor/warehouse/redshift/extract_test.py,sha256=-8eWOsFEv4DFvBmalaE_TzQD6YdgwnGRPCkKsycJuxg,653
 castor_extractor/warehouse/redshift/queries/.sqlfluff,sha256=W4pFQiY8KMtXwn3WguYQJA8cj78VR7K-iokPoZoy5aM,30
 castor_extractor/warehouse/redshift/queries/column.sql,sha256=ZXdurmaJRD2fejDksU5eh37Q4srmnVrEjSzsrtg_il8,6840
 castor_extractor/warehouse/redshift/queries/database.sql,sha256=_C0knW159YDfReGuWLjIdvxHzefo1Xg2xw2dJKJzNk8,299
 castor_extractor/warehouse/redshift/queries/group.sql,sha256=8p0wlqllnwOTiAgiV237DvFYHGOEcYwaHdyqVQg3F6E,101
 castor_extractor/warehouse/redshift/queries/query.sql,sha256=yZNGnUdebvvDx0J0KMSJ2hNgkK4gPduyOfPM_7-DIfo,3465
+castor_extractor/warehouse/redshift/queries/query_serverless.sql,sha256=QlYYFLJ2gInVczuXDxTGColM3-_zLSpPD0tBuLVFMyQ,1925
 castor_extractor/warehouse/redshift/queries/schema.sql,sha256=Mf6nooi2w2PhGxM2_kDAf3oQ8QnR-hpT5Y0AmUzghGg,585
 castor_extractor/warehouse/redshift/queries/table.sql,sha256=y8CGOwPHH_Mr8g1Zvuz2U5ldL8zuPm5v3M5RPZqIhsE,2645
 castor_extractor/warehouse/redshift/queries/table_freshness.sql,sha256=l61_ysmTEtuMwK9RmYmD5cu0HmD1RXwTEhX0ytBeyxg,726
 castor_extractor/warehouse/redshift/queries/user.sql,sha256=sEXveJAuNvZacvpI6WfwsX6VavoMb2VqYA32f6Dt-_Y,170
 castor_extractor/warehouse/redshift/queries/view_ddl.sql,sha256=Pkyh_QT6d4rhTeyiVcqw6O8CRl7NEhk2p7eM5YIn5kg,719
-castor_extractor/warehouse/redshift/query.py,sha256=0C81rkt2cpkWrJIxxwALDyqr-49vlqQM04y_N6wwStc,540
+castor_extractor/warehouse/redshift/query.py,sha256=F2MiFqPRNGfBrCtkXNRs28Q_i9DfIEKh93yDUVb8Yjw,1060
 castor_extractor/warehouse/salesforce/__init__.py,sha256=NR4aNea5jeE1xYqeZ_29deeN84CkN0_D_Z7CLQdJvFY,137
 castor_extractor/warehouse/salesforce/client.py,sha256=-9WHcQwEMrpGRQ9CN-bsRSR2Tnx9d-f_FtV4ntsf71w,3287
 castor_extractor/warehouse/salesforce/constants.py,sha256=GusduVBCPvwpk_Im6F3bDvXeNQ7hRnCMdIAjIg65RnE,52
@@ -413,8 +415,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=j_d5-HMnzBouwGfywVZMRSSwbXzPvzDWlFCZmvxcoGQ,539
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.20.4.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.20.4.dist-info/METADATA,sha256=YcFx5O-gccq_JevTWl9xfeE5LGf5baiUKHfPrG1QX28,21123
-castor_extractor-0.20.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.20.4.dist-info/entry_points.txt,sha256=IVGy_oM8VjzADMAxzmiNJTYYidTCsI98MpO_mkXjkqE,1573
-castor_extractor-0.20.4.dist-info/RECORD,,
+castor_extractor-0.20.5.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.20.5.dist-info/METADATA,sha256=wmQH2GPtnBGHq8a5CdjKZ6yPMNIrS4gkYrTpbV0T9yg,21214
+castor_extractor-0.20.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+castor_extractor-0.20.5.dist-info/entry_points.txt,sha256=IVGy_oM8VjzADMAxzmiNJTYYidTCsI98MpO_mkXjkqE,1573
+castor_extractor-0.20.5.dist-info/RECORD,,