castor-extractor 0.19.4__py3-none-any.whl → 0.19.7__py3-none-any.whl

castor_extractor/warehouse/salesforce/pagination.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from ...utils import PaginationModel
+from ...utils.salesforce.pagination import LIMIT_RECORDS_PER_PAGE
+from .soql import (
+    SOBJECTS_QUERY_TPL,
+)
+
+# Implicit (hard-coded in Salesforce) limitation when using SOQL of 2,000 rows
+FIRST_START_DURABLE_ID = "0000"
+
+
+def format_sobject_query(
+    start_durable_id: str = FIRST_START_DURABLE_ID,
+) -> str:
+    return SOBJECTS_QUERY_TPL.format(
+        start_durable_id=start_durable_id,
+        limit=LIMIT_RECORDS_PER_PAGE,
+    )
+
+
+class SalesforceSQLPagination(PaginationModel):
+    records: list
+
+    def next_page_payload(self) -> Optional[dict]:
+        start_durable_id = self.records[-1]["DurableId"]
+        query = format_sobject_query(start_durable_id)
+        return {"q": query}
+
+    def is_last(self) -> bool:
+        return len(self.records) < LIMIT_RECORDS_PER_PAGE
+
+    def page_results(self) -> list:
+        return self.records

castor_extractor/warehouse/sqlserver/client.py

@@ -1,7 +1,5 @@
 from typing import Iterator
 
-# this import is necessary so deptry does not mark the package as unused
-import pymssql  # type: ignore # noqa: F401
 from sqlalchemy import text
 
 from ...utils import ExtractionQuery, SqlalchemyClient, uri_encode

{castor_extractor-0.19.4.dist-info → castor_extractor-0.19.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.19.4
+Version: 0.19.7
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -205,8 +205,21 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 [Castor](https://castordoc.com) helps you find, understand, use your data assets
 
+
 # Changelog
 
+## 0.19.7 - 2024-09-05
+
+* Metabase: Handle compatibility with older version
+
+## 0.19.6 - 2024-09-03
+
+* Metabase: Adding error handler on API call
+
+## 0.19.5 - 2024-09-02
+
+* Databricks/Salesforce: Remove deprecated client dependencies
+
 ## 0.19.4 - 2024-08-29
 
 * Tableau Pulse: extract Metrics and Subscriptions

{castor_extractor-0.19.4.dist-info → castor_extractor-0.19.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=4sBzpYnhl78o74iaaYnUXX2JJJ_1wHUodGcVGLkl3Ag,13057
+CHANGELOG.md,sha256=cJ5aAA_dulThT10jAw6hHkBGiDyrZFqN7S_78E3q1Qk,13293
 Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -56,7 +56,7 @@ castor_extractor/quality/soda/client/__init__.py,sha256=LUvHHyyL2t3mk5GORshnef4F
 castor_extractor/quality/soda/client/client.py,sha256=8JEnLJn4-m_km-JyJaq-9RNpqBxwErVDIEgYpRju5QA,1509
 castor_extractor/quality/soda/client/credentials.py,sha256=yVFIYeEv76LogOwP_JEWieRpRFyOtACj7VLhSjyPH5k,426
 castor_extractor/quality/soda/client/endpoints.py,sha256=x3B-XlnDF8NJMuk-81N72_6HA-YZEzA895khLyj0j54,228
-castor_extractor/quality/soda/client/pagination.py,sha256=HdL9gmQyURlRVl2MsjbqH_qKFhBIdskcTb5W0EwcPMI,583
+castor_extractor/quality/soda/client/pagination.py,sha256=KgCNdLLX-G9SSnNuTslw5rh7Llwl_4LDmJEuf2ylm_A,604
 castor_extractor/types.py,sha256=Hd_shbsGAknJLTrAk3SBxZeFPOlbWBXXjIscC9C7CW8,1281
 castor_extractor/uploader/__init__.py,sha256=SSRtwjg-dNoxME-RJy9G1flASiUKAC5bH1htq3CURQg,75
 castor_extractor/uploader/constant.py,sha256=yTigLHDlYwoRr6CpFIl7ReElFsQd4H-qkluMZJPWSx0,865
@@ -65,24 +65,22 @@ castor_extractor/uploader/env_test.py,sha256=ClCWWtwd2N-5ClIDUxVMeKkWfhhOTxpppsX
 castor_extractor/uploader/upload.py,sha256=c86NP4ZxWnz3Hy1iWDYd9qjJSSjZ1bLq3fxVGBIU4Rc,3238
 castor_extractor/uploader/upload_test.py,sha256=7fwstdQe7FjuwGilsCdFpEQr1qLoR2WTRUzyy93fISw,402
 castor_extractor/uploader/utils.py,sha256=Tx_i875L2vJ8btOLV3-L0UMEFiyhH8E5n0XXRyLjO0Y,793
-castor_extractor/utils/__init__.py,sha256=YIM4iPH9N6_R1CSt5bq7LYJoMFRJjz-NUa6F2LTnmaE,1488
+castor_extractor/utils/__init__.py,sha256=MReSpH6I4AQZ5WTQp752P2sP4wVVZO8MyyglN0K0VKw,1509
 castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
 castor_extractor/utils/argument_parser_test.py,sha256=wnyLFJ74iEiPxxLSbwFtckR7FIHxsFOVU38ljs9gqRA,633
-castor_extractor/utils/client/__init__.py,sha256=wvzi8AjuDmPjseafLRDJvWFjWDvwNOIYelzzphEO--Q,419
+castor_extractor/utils/client/__init__.py,sha256=h5gm8UNNCCkAqhjYK5f6BY7k0cHFOyAvkmlktqwpir0,392
 castor_extractor/utils/client/abstract.py,sha256=aA5Qcb9TwWDSMq8WpXbGkOB20hehwX2VTpqQAwV76wk,2048
-castor_extractor/utils/client/api/__init__.py,sha256=TuFJ6D9zfx_FtAWQwWf20V-3JqIHB6EHygqVV74oRrA,247
+castor_extractor/utils/client/api/__init__.py,sha256=vlG7WXznYgLTn3XyMGsyUkgRkup8FbKM14EXJ8mv-b0,264
 castor_extractor/utils/client/api/auth.py,sha256=QDLM5h1zGibLaKyATxLF0gycg01SE92G-Y69f_YBClc,1896
 castor_extractor/utils/client/api/auth_test.py,sha256=NoZYsz7bcCyWBZdMF1TaOuK-s1j09DhTRyM4GSUW_YQ,1311
-castor_extractor/utils/client/api/client.py,sha256=eMWj_JMusXSEDoixdSil6nr9j3OQNS68Eoz8QbfKmRM,3658
+castor_extractor/utils/client/api/client.py,sha256=h8cQZ4CSD0LUuNv1VC5qa_p_jFbiOUbYztJnNmpJa2Q,4231
 castor_extractor/utils/client/api/client_test.py,sha256=FM3ZxsLLfMOBn44cXX6FIgnA31-5TTNIyp9D4LBwtXE,1222
-castor_extractor/utils/client/api/pagination.py,sha256=7uPHdCujhdBf_CmLqgs6AsQ-8wrqORsXvO_AYrdoboY,1921
+castor_extractor/utils/client/api/pagination.py,sha256=Ne-4dqKrwEAETheBn14KPrGRzxsrkuCO96Z6gkCI-0Y,2437
 castor_extractor/utils/client/api/pagination_test.py,sha256=jCOgXFXrH-jrCxe2dfk80ZksJF-EtmpJPU11BGabsqk,1385
 castor_extractor/utils/client/api/safe_request.py,sha256=SeBteAK8KhBjXldIdyUpkZphf9ktjzbvBM49AXrvD0g,1686
 castor_extractor/utils/client/api/safe_request_test.py,sha256=LqS5FBxs6lLLcTkcgxIoLb6OinxShHXR5y4CWZpwmwg,2005
 castor_extractor/utils/client/api/utils.py,sha256=jr8CWf48cIp8QP1P7oZ1zg9WaGlDO3mqCWgQKdEcpyc,238
 castor_extractor/utils/client/api/utils_test.py,sha256=a5aL-pCwa74C8Ne7OT169Bjp8WPDV5Fl8MxNxAllHJg,514
-castor_extractor/utils/client/api_deprecated.py,sha256=3yPXld2yQPbwPe1RKhXU9MV9iwWCQ85zMR23A66xQAM,2418
-castor_extractor/utils/client/api_deprecated_test.py,sha256=uOmY7M62bmDBzINor5OcxVeZktI0LthVeTD6NxjGtUo,607
 castor_extractor/utils/client/postgres.py,sha256=n6ulaT222WWPY0_6qAZ0MHF0m91HtI9mMqL71nyygo0,866
 castor_extractor/utils/client/query.py,sha256=O6D5EjD1KmBlwa786Uw4D4kzxx97_HH50xIIeSWt0B8,205
 castor_extractor/utils/client/uri.py,sha256=jmP9hY-6PRqdc3-vAOdtll_U6q9VCqSqmBAN6QRs3ZI,150
@@ -107,23 +105,22 @@ castor_extractor/utils/json_stream_write.py,sha256=OUdg4-47I47pgbpN9_a6y-lmxuod7
 castor_extractor/utils/load.py,sha256=MXwGVB_Dp_VitGwo71sNB_xDmGzQ4oQ13MhaXXyYkS0,265
 castor_extractor/utils/object.py,sha256=xCcQtoj9313TCcoyRXkLpDcMxmDeQMFMseDNx95oGc0,1959
 castor_extractor/utils/object_test.py,sha256=uy8p1VTtJX5Q1AhpORRlx7rM6vLuhnW0Gb3fprBIlG8,2487
-castor_extractor/utils/pager/__init__.py,sha256=IoZnJVkALgEAg5P_fiecrPbukLFsKT8AmL7qA8mov0g,165
+castor_extractor/utils/pager/__init__.py,sha256=whWh-Kr524ELXrukzorPWo-7ugEPJTcn-vzb9F096J8,124
 castor_extractor/utils/pager/pager.py,sha256=h_toZkR6RbXh-rdbAKule86bVijEqriV3GMnMTpCYIM,2389
 castor_extractor/utils/pager/pager_on_id.py,sha256=WATgZcZ2_1W2v1IbkwKIwjw2paoimm8r-ZJiNtPVc5w,1940
 castor_extractor/utils/pager/pager_on_id_test.py,sha256=CfAXhXaAmCXnm0oflj8_82An6znOS2mBk4nqvtrCqs4,1613
-castor_extractor/utils/pager/pager_on_token.py,sha256=G442SKl4BXJFMPbYIIgCk5M8wl7V3jMg3K1WUUkl0I0,1579
-castor_extractor/utils/pager/pager_on_token_test.py,sha256=w2GCUGKR3cD5lfmtFAsNvExtzxkYdBR0pusBrGKFQ08,2548
 castor_extractor/utils/pager/pager_test.py,sha256=QPBVShSXhkiYZUfnAMs43xnys6CD8pAhL3Jhj-Ov2Xc,1705
 castor_extractor/utils/retry.py,sha256=pTCASEtKjSBPwS5ummaCF_C0rZcQwEhOPYRQi0PpDG0,4719
 castor_extractor/utils/retry_test.py,sha256=YMZfiEy95FQHsp8StPDAHfMvljIl-nLNpAjwRFh91KA,2916
 castor_extractor/utils/safe.py,sha256=jpfIimwdBSVUvU2DPFrhqpKC_DSYwxQqd08MlIkSODY,1967
 castor_extractor/utils/safe_test.py,sha256=IHN1Z761tYMFslYC-2HAfkXmFPh4LYSqNLs4QZwykjk,2160
 castor_extractor/utils/salesforce/__init__.py,sha256=fZ2U6t6AFFAIC-DLXvFHBgpBDjTvX0tFgZ8zJoehPAc,88
-castor_extractor/utils/salesforce/client.py,sha256=nvNikpuQAJJPRnpARKeeFzUK0MbPCf8XRAKlFev-tgQ,2877
-castor_extractor/utils/salesforce/client_test.py,sha256=s6UTogjC36jrJOnYA-gFuyTQsvROCt9y_eoD2O41xCg,682
+castor_extractor/utils/salesforce/client.py,sha256=aX_iWQFWdhk8gh16_gSHYeCI9A7wqohcQhY-DOEWKko,2350
+castor_extractor/utils/salesforce/client_test.py,sha256=T3gUnZ0cRvnL_4dVc4lInRSO9Ti2WeLkLWV1scps4IY,668
 castor_extractor/utils/salesforce/constants.py,sha256=7yPmUeyn4IHQiHLDutXE0L_OBd41E5080vFxqA_s4Dc,58
 castor_extractor/utils/salesforce/credentials.py,sha256=Fp-7yaidGpeuaXad1ng6dr1jSOI9Bl2KMVZXPyTBVjM,1148
 castor_extractor/utils/salesforce/credentials_test.py,sha256=FQRyNk2Jsh6KtYiW20oL43CVnGjXLcAjdFATkE7jK0s,586
+castor_extractor/utils/salesforce/pagination.py,sha256=wJq0rKLdacFRggyHwB6Fh3K6iXPvL4QWhsDvZdjQjM8,849
 castor_extractor/utils/store.py,sha256=D_pVaPsu1MKAJC0K47O_vYTs-Afl6oejravAJdvjmGc,2040
 castor_extractor/utils/string.py,sha256=aW6bbjqEGnh9kT5KZBnMlV6fhdgOJ0ENCkCTDon1xA0,2377
 castor_extractor/utils/string_test.py,sha256=QWuNog77GogQsPI5kSXLskq6z3Obt89YbugWEia6PH4,2527
@@ -164,7 +161,7 @@ castor_extractor/visualization/metabase/__init__.py,sha256=3E36cmkMyEgBB6Ot5rWk-
 castor_extractor/visualization/metabase/assets.py,sha256=RrrE7H0ezqD-YpNLjDOo4u5fFPUK1PIRxLXNERFqFn8,2906
 castor_extractor/visualization/metabase/client/__init__.py,sha256=KBvaPMofBRV3m_sZAnKNCrJGr-Z88EbpdzEzWPQ_uBk,99
 castor_extractor/visualization/metabase/client/api/__init__.py,sha256=BYSPWHY4KbT-LvenNI0pRonxolzZ5HSl6v3-PbJr-7M,78
-castor_extractor/visualization/metabase/client/api/client.py,sha256=sQIROc8lKEEUs496Qhi_Uvk9YV1Is4klqmn6Etvudbs,5935
+castor_extractor/visualization/metabase/client/api/client.py,sha256=2b8NUBjVev-DcMG-rshCPGbG4673NRgjgseUy46JI5M,6460
 castor_extractor/visualization/metabase/client/api/client_test.py,sha256=7Lb5yvrvHQmCIOnFzS2D00oR2Zps1SVxGIAukzRVeKg,559
 castor_extractor/visualization/metabase/client/api/credentials.py,sha256=SiHWyWWEDy_Ak4ne0hgXPZCSP6mEuHH26wDhsed_9v8,519
 castor_extractor/visualization/metabase/client/db/__init__.py,sha256=nawDhJ-JGlpM6VMzZZRjf066QXk9kWzZr6l9n6OHTZ0,76
@@ -226,13 +223,13 @@ castor_extractor/visualization/qlik/extract.py,sha256=2ACqmWheVdkIXmDtmhdunf8oyN
 castor_extractor/visualization/salesforce_reporting/__init__.py,sha256=MvArD0GKNIpCDvLIYcpKrjMjFLhMyDETK6i3k0Fb6Tk,124
 castor_extractor/visualization/salesforce_reporting/assets.py,sha256=2J-iAmJGGDufOcJUgE47M3-dEcjYXcVyVUNcmHrj79w,271
 castor_extractor/visualization/salesforce_reporting/client/__init__.py,sha256=DIA6f_vNJZqT89qVYxg98Le7QeDn2y0Qew03V3J9t9o,44
-castor_extractor/visualization/salesforce_reporting/client/rest.py,sha256=hzaXWLcYt0aAHXK46DbsLmzocjRY1llwrNj8_3TObKs,1849
+castor_extractor/visualization/salesforce_reporting/client/rest.py,sha256=97ud629qekagQvInQnTk-Ni0mFQKxJWyeYAeKO8XA6I,1866
 castor_extractor/visualization/salesforce_reporting/client/soql.py,sha256=DcJgchC9IYcK_bLdwskwycItCh3wtfKkwj4nyaSn66Q,1646
 castor_extractor/visualization/salesforce_reporting/extract.py,sha256=RMhlf7NeYiAO_8hkFk4-JxTghNbC0vtX3P2M80qt9U8,1339
 castor_extractor/visualization/sigma/__init__.py,sha256=GINql4yJLtjfOJgjHaWNpE13cMtnKNytiFRomwav27Q,114
 castor_extractor/visualization/sigma/assets.py,sha256=JZ1Cpxnml8P3mIJoTUM57hvylB18ErECQXaP5FF63O4,268
 castor_extractor/visualization/sigma/client/__init__.py,sha256=YQv06FBBQHvBMFg_tN0nUcmUp2NCL2s-eFTXG8rXaBg,74
-castor_extractor/visualization/sigma/client/client.py,sha256=RKFghxdlznAHvuBayg2SvgpqTN5pvtaeNEGbAnfbrR8,6189
+castor_extractor/visualization/sigma/client/client.py,sha256=Gi8GzP_xxjDZDmNRK4pXJUvSPzZCCJpn6ClUTML5amE,6218
 castor_extractor/visualization/sigma/client/credentials.py,sha256=0f2VeDFaNbaZOTN0fPdmx3s-nxxFe35yBF9D_2B2lGI,718
 castor_extractor/visualization/sigma/client/endpoints.py,sha256=DBFphbgoH78_MZUGM_bKBAq28Nl7LWSZ6VRsbxrxtDg,1162
 castor_extractor/visualization/sigma/client/pagination.py,sha256=gHBewkYoJaR0vADzU_VPsNkcAa-6KAC6hplpDL5gT8U,690
@@ -273,9 +270,9 @@ castor_extractor/visualization/tableau/tsc_fields.py,sha256=BoV6XVu-HUan9hxeYRbv
 castor_extractor/visualization/tableau/types.py,sha256=_T3fahMHRkV2TVaYcjITh61T0FLzSBYKD21VurzkF5M,322
 castor_extractor/visualization/tableau/usage.py,sha256=LlFwlbEr-EnYUJjKZha99CRCRrERJ350oAvzBQlp9_s,427
 castor_extractor/visualization/tableau_revamp/__init__.py,sha256=a3DGjQhaz17gBqW-E84TAgupKbqLC40y5Ajo1yn-ot4,156
-castor_extractor/visualization/tableau_revamp/assets.py,sha256=HTJ6OyCkUdhv73VI2AlZFumat9kIz7T1dMJ4ljB_I6A,455
+castor_extractor/visualization/tableau_revamp/assets.py,sha256=8sJsK6Qixao6xVmVaO1usvs16SjNub9sIx7o-adYV14,659
 castor_extractor/visualization/tableau_revamp/client/__init__.py,sha256=wmS9uLtUiqNYVloi0-DgD8d2qzu3RVZEAtWiaDp6G_M,90
-castor_extractor/visualization/tableau_revamp/client/client.py,sha256=_vlTckFRPsSQzIaLUM2RrXabHgFrWduV61Z8L8OSjA0,6902
+castor_extractor/visualization/tableau_revamp/client/client.py,sha256=8pATeVTKslF5HCbN1c_DusysP7lTU51yXPp8uhzWwZg,7165
 castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py,sha256=yNnGR3Tk32TUmaDejaz5fkw2p9DtmMeCv5rsZNOHUfY,3047
 castor_extractor/visualization/tableau_revamp/client/client_rest_api.py,sha256=0g8AddrhzirRCUWR2jrudPR02mk4Of5YilWth7zJO-g,4016
 castor_extractor/visualization/tableau_revamp/client/client_tsc.py,sha256=BBwIOqK2zU66udFRmLGmB_3J1ILGhVOY5Hq4nmsonF0,1853
@@ -311,18 +308,25 @@ castor_extractor/warehouse/bigquery/queries/view_ddl.sql,sha256=obCm-IN9V8_YSZTw
 castor_extractor/warehouse/bigquery/query.py,sha256=5Qc8PEa-kQKpTzguj4RNCAwKyvzWt20vAESYNB0lueo,4768
 castor_extractor/warehouse/bigquery/types.py,sha256=DHK3wUaaLyLMp7LP-7QkXTDYpYTZiPtvptAOkpxgp4g,88
 castor_extractor/warehouse/databricks/__init__.py,sha256=YG3YSIJgCFRjjI8eExy9T7qGnfnjWhMFh8c15KTs_BA,184
-castor_extractor/warehouse/databricks/client.py,sha256=lBIRMHucLWJzyk3TXKdT8W3adzr4W9r53W1gjzyA8hs,21434
-castor_extractor/warehouse/databricks/client_test.py,sha256=DpjkcvMbli-SYRNI-41VWG0xtQ4trjDoJBMxKs4CrPU,5625
+castor_extractor/warehouse/databricks/api_client.py,sha256=lfS-DMLlTEsEMAEs7TeH8JmLX74l4Ai0k12nuP2fo2Y,8110
+castor_extractor/warehouse/databricks/api_client_test.py,sha256=YTWC-X7L-XAfK5b39TUgTmR1ifv0QrY5tvLNoSbpmjg,466
+castor_extractor/warehouse/databricks/client.py,sha256=neH-KnyWtxEMApbjIV2uhY7Me3tCpavd1NMnljrT05k,4768
+castor_extractor/warehouse/databricks/client_test.py,sha256=UKr_D3M8mhqV1oL2_3y_6pEzAFLVE3FHDNZh4omFLK4,2286
 castor_extractor/warehouse/databricks/credentials.py,sha256=iphbVynVTQXMEbJy4QaT5fer-GpOi7QtbAlg8R7-Lj4,598
+castor_extractor/warehouse/databricks/endpoints.py,sha256=qPoL9CtPFJdwVuW9rJ37nmeMd-nChOBouEVYb4SlaUE,670
 castor_extractor/warehouse/databricks/extract.py,sha256=VYygE06f7ngYWVlRa48O6drLIZF-_4IBJdyXTYfxZQU,7395
 castor_extractor/warehouse/databricks/format.py,sha256=p252NFzQN1uZdsu5wpP-bMHK0rBBVzallX3-o92Mvh4,6744
 castor_extractor/warehouse/databricks/format_test.py,sha256=ls0IcOElqp_qecAzNbK0zdca7Pms4seCHimbw8NAoAI,3322
+castor_extractor/warehouse/databricks/lineage.py,sha256=qWVXCT9bRYVYPV5hsiUgPcdThsMN-6EeCEDKL8Fykj0,4724
+castor_extractor/warehouse/databricks/lineage_test.py,sha256=EejO4qKH_kJlJSrIap6GvkUi9E55RFvfiySKazAh0_A,1048
+castor_extractor/warehouse/databricks/pagination.py,sha256=pYxdoUBEGkd1p_zQRme60NEn3dBp7H0VxQO3la-gOmc,551
+castor_extractor/warehouse/databricks/sql_client.py,sha256=KBP0rmMQBWw3jshDfv_NpFW8HqPxGfcBkS4d9T9aXvE,2977
 castor_extractor/warehouse/databricks/test_constants.py,sha256=Hm96yq_ltVAKv7WYhYz637r4Cuj-1cCdyOuxMEe3J-Q,2246
 castor_extractor/warehouse/databricks/types.py,sha256=hD6gC8oiT3QSWEvbtgUOGK_lLzzz36sEauB3lS_wxlE,218
-castor_extractor/warehouse/databricks/utils.py,sha256=RWRViqLaj2K0in5T5F6OLp7HCm554BCh3zi4CJqOEt8,576
-castor_extractor/warehouse/databricks/utils_test.py,sha256=5Qrd_tLNLWrDHX2uQyVUf0vqXJzD44uQGGxDBOkwvUU,503
+castor_extractor/warehouse/databricks/utils.py,sha256=QOYpv50rqxUrr302vdzXJlBRT4CZV8zb7gtmW9g5n2E,1954
+castor_extractor/warehouse/databricks/utils_test.py,sha256=_guTuzRWRTZdDY7ils0X1K8jhI9T877MEtw3x_YDg9I,2415
 castor_extractor/warehouse/mysql/__init__.py,sha256=2KFDogo9GNbApHqw3Vm5t_uNmIRjdp76nmP_WQQMfQY,116
-castor_extractor/warehouse/mysql/client.py,sha256=TgC7HfEReKOCO0ynJ3YqoJqmbRj9zaLnMjaRynOg7ao,1060
+castor_extractor/warehouse/mysql/client.py,sha256=DUDI8Zm24kxYjh9xYgaRZUj8K_JlPjcjr67gmGfFpNs,942
 castor_extractor/warehouse/mysql/client_test.py,sha256=wRTv-3c5chy_HKj-buasNiYOOCIfynYqbabM4Hxdh5E,1052
 castor_extractor/warehouse/mysql/extract.py,sha256=nGDVA1DrMBQEuUBFrNwnzZbUR0SZeg0a8QN2Aemfnto,2124
 castor_extractor/warehouse/mysql/queries/.sqlfluff,sha256=RjzW9cyXVYL1l55HpaC3zFren7CIh9lVGwoODNk2i-4,27
@@ -360,11 +364,12 @@ castor_extractor/warehouse/redshift/queries/user.sql,sha256=sEXveJAuNvZacvpI6Wfw
 castor_extractor/warehouse/redshift/queries/view_ddl.sql,sha256=Pkyh_QT6d4rhTeyiVcqw6O8CRl7NEhk2p7eM5YIn5kg,719
 castor_extractor/warehouse/redshift/query.py,sha256=0C81rkt2cpkWrJIxxwALDyqr-49vlqQM04y_N6wwStc,540
 castor_extractor/warehouse/salesforce/__init__.py,sha256=NR4aNea5jeE1xYqeZ_29deeN84CkN0_D_Z7CLQdJvFY,137
-castor_extractor/warehouse/salesforce/client.py,sha256=F3UdD5-9umEU-u_c7uVtaksg81VZeXRW83BVsFvYDkE,4902
+castor_extractor/warehouse/salesforce/client.py,sha256=NbbXTi_eX0ge815FgsiWh4uUnvZMOAl9_mXA_e172_0,3281
 castor_extractor/warehouse/salesforce/constants.py,sha256=GusduVBCPvwpk_Im6F3bDvXeNQ7hRnCMdIAjIg65RnE,52
 castor_extractor/warehouse/salesforce/extract.py,sha256=GaxkGWhdksDT_rlT24KX8DMpWnhKlhDMAUvBPGalli0,3454
 castor_extractor/warehouse/salesforce/format.py,sha256=eiPM_4i_m3FEg_2jkMYlhaRBg3gTvV-9xQuk8ghJZiM,3289
 castor_extractor/warehouse/salesforce/format_test.py,sha256=6xDtCxNqvLo5JeHCtXyAun62WMzfVaVsvvMGXXfGgmA,2254
+castor_extractor/warehouse/salesforce/pagination.py,sha256=m1S9JRNf6Oe-6dDghYUY5wwTzGzKW5H9pE60PCXMha0,920
 castor_extractor/warehouse/salesforce/soql.py,sha256=XB8ohKwHFfC4Xger7Y84DXLW17IJDye_bZ3FL6DCcOI,1188
 castor_extractor/warehouse/snowflake/__init__.py,sha256=TEGXTyxWp4Tr9gIHb-UFVTRKj6YWmrRtqHruiKSZGiY,174
 castor_extractor/warehouse/snowflake/client.py,sha256=XT0QLVNff_586SDuMe40iu8FCwPDh2uBV5aKc1Ql914,5555
@@ -387,7 +392,7 @@ castor_extractor/warehouse/snowflake/queries/user.sql,sha256=88V8eRj1NDaD_ufclsK
 castor_extractor/warehouse/snowflake/queries/view_ddl.sql,sha256=eWsci_50cxiYIv3N7BKkbXVM3RoIzqSDtohqRnE5kg4,673
 castor_extractor/warehouse/snowflake/query.py,sha256=yDpG4e23xtjEfAKNSAgL9wx17ChFSlvAbig2mJ5ZEC0,1769
 castor_extractor/warehouse/sqlserver/__init__.py,sha256=PdOuYznmvKAbfWAm8UdN47MfEsd9jqPi_dDi3WEo1KY,116
-castor_extractor/warehouse/sqlserver/client.py,sha256=MpjfIlUvWYNuHJmnFsd7rBYdGllAJPuk7CZKuSE_E5w,1709
+castor_extractor/warehouse/sqlserver/client.py,sha256=l6O0OhAKtRqy0oTl9KDYDawgWKc4XCNOA-bVdnMdEjM,1592
 castor_extractor/warehouse/sqlserver/extract.py,sha256=2mBNx9clyrhoiirD635BW-5u6pPoxHyIsB071XoZjho,2087
 castor_extractor/warehouse/sqlserver/queries/.sqlfluff,sha256=yy0KQdz8I_67vnXyX8eeWwOWkxTXvHyVKSVwhURktd8,48
 castor_extractor/warehouse/sqlserver/queries/column.sql,sha256=Szdf8hwcDffRTgtD6zf4ZuIyHIVijFgSDk1rZbKI3g8,2480
@@ -406,8 +411,8 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.19.4.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.19.4.dist-info/METADATA,sha256=1jzZUZIN0akVRTWJNZ0HLBCKikI-3cPHAO3kabMNBhw,20275
-castor_extractor-0.19.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.19.4.dist-info/entry_points.txt,sha256=X_pDYOmhUUMbiAD9h2GZveuGdT8UgL38KJqP44xkvqo,1495
-castor_extractor-0.19.4.dist-info/RECORD,,
+castor_extractor-0.19.7.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.19.7.dist-info/METADATA,sha256=C94_SY5XYZLhVttCVc8CHNOIKny9F9mTiDpq_dBt79A,20511
+castor_extractor-0.19.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+castor_extractor-0.19.7.dist-info/entry_points.txt,sha256=X_pDYOmhUUMbiAD9h2GZveuGdT8UgL38KJqP44xkvqo,1495
+castor_extractor-0.19.7.dist-info/RECORD,,

castor_extractor/utils/client/api_deprecated.py

@@ -1,89 +0,0 @@
-import logging
-from typing import Any, Callable, Dict, Literal, Optional
-
-import requests
-
-from .api.safe_request import (
-    RequestSafeMode,
-    handle_response,
-)
-
-logger = logging.getLogger(__name__)
-
-DEFAULT_TIMEOUT_S = 30
-
-# https://requests.readthedocs.io/en/latest/api/#requests.request
-HttpMethod = Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]
-
-
-def _authentication_header(token: Optional[str] = None) -> Dict[str, str]:
-    if token:
-        return {"Authorization": f"Bearer {token}"}
-    return dict()
-
-
-class APIClientDeprecated:
-    """
-    API client
-    - authentication via access token
-    """
-
-    def __init__(
-        self,
-        host: str,
-        token: Optional[str] = None,
-        headers: Optional[Dict[str, str]] = None,
-        timeout: int = DEFAULT_TIMEOUT_S,
-        safe_mode: RequestSafeMode = RequestSafeMode(),
-    ):
-        self._host = host
-        self._token = token or ""
-        self._timeout = timeout
-        self._base_headers = headers or {}
-        self.safe_mode = safe_mode
-
-    @staticmethod
-    def build_url(host: str, path: str):
-        if not host.startswith("https://"):
-            host = "https://" + host
-        return f"{host.strip('/')}/{path}"
-
-    @property
-    def _headers(self):
-        """Returns specified headers and authentication headers altogether"""
-        return {**self._base_headers, **_authentication_header(self._token)}
-
-    def _call(
-        self,
-        url: str,
-        method: HttpMethod = "GET",
-        *,
-        params: Optional[dict] = None,
-        data: Optional[dict] = None,
-        processor: Optional[Callable] = None,
-    ) -> Any:
-        logger.debug(f"Calling {method} on {url}")
-        response = requests.request(
-            method,
-            url,
-            headers=self._headers,
-            params=params,
-            json=data,
-            timeout=self._timeout,
-        )
-        response_payload = handle_response(response, self.safe_mode)
-
-        if processor:
-            return processor(response)
-
-        return response_payload
-
-    def get(
-        self,
-        path: str,
-        payload: Optional[dict] = None,
-        processor: Optional[Callable] = None,
-    ) -> dict:
-        """path: REST API operation path, such as /api/2.0/clusters/get"""
-        url = self.build_url(self._host, path)
-        return self._call(url=url, data=payload, processor=processor)

castor_extractor/utils/client/api_deprecated_test.py

@@ -1,18 +0,0 @@
-from .api_deprecated import APIClientDeprecated
-
-
-def test_APIClient_build_url():
-    expected = "https://3.14.azuredatabricks.net/api/2.1/unity-catalog/tables"
-
-    path = "api/2.1/unity-catalog/tables"
-
-    host = "3.14.azuredatabricks.net"
-    assert expected == APIClientDeprecated.build_url(host, path)
-
-    host_with_http = "https://3.14.azuredatabricks.net"
-    assert expected == APIClientDeprecated.build_url(host_with_http, path)
-
-    host_with_trailing_slash = "https://3.14.azuredatabricks.net/"
-    assert expected == APIClientDeprecated.build_url(
-        host_with_trailing_slash, path
-    )

castor_extractor/utils/pager/pager_on_token.py

@@ -1,52 +0,0 @@
-from typing import Any, Callable, Dict, Iterator, Optional, Sequence, TypeVar
-
-from .pager import (
-    DEFAULT_PER_PAGE,
-    AbstractPager,
-    PagerLogger,
-    PagerStopStrategy,
-)
-
-T = TypeVar("T")
-
-
-class PagerOnToken(AbstractPager):
-    """Token-based pagination"""
-
-    def __init__(
-        self,
-        callback: Callable[[Optional[str], Optional[int]], Dict[Any, Any]],
-        *,
-        logger: Optional[PagerLogger] = None,
-        start_page: int = 1,
-        stop_strategy: PagerStopStrategy = PagerStopStrategy.EMPTY_PAGE,
-    ):
-        self._callback = callback
-        self._logger = logger or PagerLogger()
-        self._start_page = start_page
-        self._stop_strategy = stop_strategy
-
-    def iterator(
-        self,
-        per_page: int = DEFAULT_PER_PAGE,
-    ) -> Iterator[Sequence[T]]:
-        """Yields data provided by the callback as a list using the nexttoken"""
-        stop_on_empty_page = self._stop_strategy == PagerStopStrategy.EMPTY_PAGE
-        page = self._start_page
-        total_results = 0
-        page_token = None
-        while True:
-            results = self._callback(page_token, per_page)
-            nb_results = len(results.get("res", []))
-            total_results += nb_results
-
-            if results.get("res"):
-                yield results["res"]
-
-            stop = self.should_stop(nb_results, per_page, stop_on_empty_page)
-            page_token = results.get("next_page_token")
-            if stop or not page_token:
-                break
-            page += 1
-
-        self._logger.on_success(page, total_results)

castor_extractor/utils/pager/pager_on_token_test.py

@@ -1,73 +0,0 @@
-from typing import Any, Callable, Dict, List, Optional
-
-from .pager_on_token import PagerOnToken
-
-ITEMS = list(range(1000))
-
-OStr = Optional[str]
-
-
-def _make_callback_with_token(
-    elements: List[Dict[str, str]],
-) -> Callable[[OStr, int], Dict[str, Any]]:
-    def _callback(page_token: OStr, per_page: int) -> Dict[str, Any]:
-        """callable with a token that indicates how to retrieve the next page"""
-        if page_token:
-            _start = int(page_token)
-        else:
-            _start = 0
-        _end = _start + per_page
-        return {"res": elements[_start:_end], "next_page_token": str(_end)}
-
-    return _callback
-
-
-def _make_callback_with_missing_token(
-    elements: List[Dict[str, str]],
-) -> Callable[[OStr, int], Dict[str, Any]]:
-    def _callback(page_token: OStr, per_page: int) -> Dict[str, Any]:
-        """callable with a token that indicates how to retrieve the next page
-        except for the last page"""
-        if page_token:
-            _start = int(page_token)
-        else:
-            _start = 0
-        _end = _start + per_page
-        if _end == len(ITEMS):
-            return {"res": elements[_start:_end]}
-        return {"res": elements[_start:_end], "next_page_token": str(_end)}
-
-    return _callback
-
-
-def test_pagerontoken__all():
-    """unit test for PagerOnToken#all()"""
-    pager = PagerOnToken(_make_callback_with_token(ITEMS))
-    # When no argument provided
-    assert pager.all() == ITEMS
-    # When per page is less than the number of ITEMS
-    assert pager.all(per_page=1) == ITEMS
-    # When per page is more than the number of ITEMS
-    assert pager.all(per_page=len(ITEMS) + 20) == ITEMS
-
-    # Same test suite, but no token is provided at the last call
-    pager = PagerOnToken(_make_callback_with_missing_token(ITEMS))
-    # When no argument provided
-    assert pager.all() == ITEMS
-    # When per page is less than the number of ITEMS
-    assert pager.all(per_page=1) == ITEMS
-    # When per page is more than the number of ITEMS
-    assert pager.all(per_page=len(ITEMS) + 20) == ITEMS
-
-
-def test_pagerontoken__iterator__pagination():
-    """unit test for PagerOnToken#iterator() (pagination)"""
-    pager = PagerOnToken(_make_callback_with_token(ITEMS))
-
-    def nb_of_pages(per_page: int) -> int:
-        return len([page for page in pager.iterator(per_page=per_page)])
-
-    assert nb_of_pages(per_page=len(ITEMS)) == 1
-    assert nb_of_pages(per_page=1) == len(ITEMS)
-    assert nb_of_pages(per_page=2) == len(ITEMS) // 2
-    assert nb_of_pages(per_page=4) == len(ITEMS) // 4