castor-extractor 0.20.0__py3-none-any.whl → 0.20.4__py3-none-any.whl

castor_extractor/warehouse/salesforce/format_test.py

@@ -1,15 +1,15 @@
-from typing import Dict, Tuple
+from typing import Dict, List, Tuple
 
 from .format import (
-    SCHEMA_NAME,
+    _HAS_DUPLICATE_KEY,
     SalesforceFormatter,
-    _by_label,
+    _detect_duplicates,
     _field_description,
-    _merge_label_and_api_name,
+    _name,
 )
 
 
-def _example_sobjects() -> Tuple[Dict[str, str], ...]:
+def _tables_sobjects() -> Tuple[Dict[str, str], ...]:
     """Returns 4 sobjects with 2 sharing the same label"""
     a = {"Label": "a", "QualifiedApiName": "a_one"}
     b = {"Label": "b", "QualifiedApiName": "b"}
@@ -18,6 +18,16 @@ def _example_sobjects() -> Tuple[Dict[str, str], ...]:
     return a, b, c, a_prime
 
 
+def _columns_sobjects() -> Dict[str, List[dict]]:
+    a = {"Label": "First Name", "QualifiedApiName": "owner_name"}
+    b = {"Label": "First Name", "QualifiedApiName": "editor_name"}
+    c = {"Label": "Foo Bar", "QualifiedApiName": "foo_bar"}
+    return {
+        "table_1": [a, b],
+        "table_2": [c],
+    }
+
+
 def test__field_description():
     field = {}
     assert _field_description(field) == ""
@@ -48,32 +58,59 @@ def test__field_description():
     assert description == expected
 
 
-def test__merge_label_and_api_name():
-    sobject = {"Label": "foo", "QualifiedApiName": "bar"}
-    payload = _merge_label_and_api_name(sobject)
-    expected_name = "foo (bar)"
-    assert payload == {
-        "id": expected_name,
-        "api_name": "bar",
-        "label": "foo",
-        "schema_id": SCHEMA_NAME,
-        "table_name": expected_name,
-        "description": None,
-        "tags": [],
-        "type": "TABLE",
+def test__name():
+    unique_sobject = {
+        "Label": "First Name",
+        "QualifiedApiName": "first_name",
+        _HAS_DUPLICATE_KEY: False,
     }
+    assert _name(unique_sobject) == "First Name"
 
+    duplicate_sobject = {
+        "Label": "First Name",
+        "QualifiedApiName": "first_name",
+        _HAS_DUPLICATE_KEY: True,
+    }
+    assert _name(duplicate_sobject) == "First Name (first_name)"
+
+    empty_label_sobject = {
+        "Label": "",
+        "QualifiedApiName": "empty_label",
+        _HAS_DUPLICATE_KEY: False,
+    }
+    assert _name(empty_label_sobject) == "empty_label"
 
-def test__by_label():
-    a, b, c, a_prime = _example_sobjects()
-    sobjects = [a, b, c, a_prime]
-    by_label = _by_label(sobjects)
-    assert by_label == {"a": [a, a_prime], "b": [b], "c": [c]}
+
+def test__detect_duplicates():
+    objects = [
+        {"Label": "Foo"},
+        {"Label": "Bar"},
+        {"Label": "Foo"},
+    ]
+
+    objects = _detect_duplicates(objects)
+    assert objects == [
+        {"Label": "Foo", _HAS_DUPLICATE_KEY: True},
+        {"Label": "Bar", _HAS_DUPLICATE_KEY: False},
+        {"Label": "Foo", _HAS_DUPLICATE_KEY: True},
+    ]
 
 
 def test_salesforce_formatter_tables():
-    sobjects = [*_example_sobjects()]
+    sobjects = [*_tables_sobjects()]
     tables = SalesforceFormatter.tables(sobjects)
     expected_names = {"a (a_one)", "a (a_two)", "b", "c"}
     payload_names = {t["table_name"] for t in tables}
     assert payload_names == expected_names
+
+
+def test_salesforce_formatter_columns():
+    sobjects = _columns_sobjects()
+    columns = SalesforceFormatter.columns(sobjects)
+    column_ids = {c["id"] for c in columns}
+    expected_column_ids = {
+        "table_1.First Name (owner_name)",
+        "table_1.First Name (editor_name)",
+        "table_2.Foo Bar",
+    }
+    assert column_ids == expected_column_ids

{castor_extractor-0.20.0.dist-info → castor_extractor-0.20.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.20.0
+Version: 0.20.4
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -208,6 +208,22 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
 
 # Changelog
 
+## 0.20.4 - 2024-10-09
+
+* Salesforce warehouse: `Labels` instead of `api_names` for columns
+
+## 0.20.3 - 2024-10-03
+
+* Looker: no longer extract `as_html` dashboard elements
+
+## 0.20.2 - 2024-09-24
+
+* Thoughtspot: Adding connector
+
+## 0.20.1 - 2024-09-23
+
+* Power BI: Improved client based on APIClient
+
 ## 0.20.0 - 2024-09-23
 
 * Switch to Tableau revamped connector

{castor_extractor-0.20.0.dist-info → castor_extractor-0.20.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=_iTzy8VrdNYmYKWXfGfPMIlSenr4M7LCvoE1K0H96co,13601
+CHANGELOG.md,sha256=CzVaQbFAS2hlZE2ak7DTYHWBNjMaC59e8UK7Q9p10tw,13905
 Dockerfile,sha256=HcX5z8OpeSvkScQsN-Y7CNMUig_UB6vTMDl7uqzuLGE,303
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -24,6 +24,7 @@ castor_extractor/commands/extract_sigma.py,sha256=sxewHcZ1Doq35V2qnpX_zCKKXkrb1_
 castor_extractor/commands/extract_snowflake.py,sha256=vYiruxRoo--GeMemOGsSE1w9kcKTh_y4E165HtMVzkM,1982
 castor_extractor/commands/extract_sqlserver.py,sha256=lwhbcNChaXHZgMgSOch3faVr7WJw-sDU6GHl3lzBt_0,1141
 castor_extractor/commands/extract_tableau.py,sha256=VUb_1Y85EzfF1f9OaCQQt8kFYBdp0u31Mw1Wm2fkxWs,1221
+castor_extractor/commands/extract_thoughtspot.py,sha256=caAYJlH-vK7u5IUB6OKXxcaWfLgc7d_XqnFDWK6YNS4,639
 castor_extractor/commands/file_check.py,sha256=VSD84kpQKf7b0wJOhUgkJQ9n4mK3v52sjMWL7wkNYa0,2667
 castor_extractor/commands/upload.py,sha256=WLDI3zDmK2CjtbxiMWX2mZGjxx8DozfCw6tLE3CAMcE,1833
 castor_extractor/file_checker/__init__.py,sha256=OSt6YLhUT42U_Cp3LCLHMVruwDkksL75Ij13X2UPnVk,119
@@ -73,11 +74,11 @@ castor_extractor/utils/client/abstract.py,sha256=aA5Qcb9TwWDSMq8WpXbGkOB20hehwX2
 castor_extractor/utils/client/api/__init__.py,sha256=vlG7WXznYgLTn3XyMGsyUkgRkup8FbKM14EXJ8mv-b0,264
 castor_extractor/utils/client/api/auth.py,sha256=QDLM5h1zGibLaKyATxLF0gycg01SE92G-Y69f_YBClc,1896
 castor_extractor/utils/client/api/auth_test.py,sha256=NoZYsz7bcCyWBZdMF1TaOuK-s1j09DhTRyM4GSUW_YQ,1311
-castor_extractor/utils/client/api/client.py,sha256=0E5GG5Yxk-J5B11YdeIcccYk7jAfuQoWJIz5ljMGYUE,4275
+castor_extractor/utils/client/api/client.py,sha256=_XkOMbkit0_YYJ2tY1rYrEuoPTUP826CJPnI9OLgmuQ,4434
 castor_extractor/utils/client/api/client_test.py,sha256=FM3ZxsLLfMOBn44cXX6FIgnA31-5TTNIyp9D4LBwtXE,1222
 castor_extractor/utils/client/api/pagination.py,sha256=Efg3P9ct_U5rtgXijMGV05oQxSzjldEopECWjIFWerM,2439
 castor_extractor/utils/client/api/pagination_test.py,sha256=jCOgXFXrH-jrCxe2dfk80ZksJF-EtmpJPU11BGabsqk,1385
-castor_extractor/utils/client/api/safe_request.py,sha256=SeBteAK8KhBjXldIdyUpkZphf9ktjzbvBM49AXrvD0g,1686
+castor_extractor/utils/client/api/safe_request.py,sha256=dh69Uv9LMUGKDnxDnBYEYMEdTWq2GHLWD3ZwXtQP3So,1787
 castor_extractor/utils/client/api/safe_request_test.py,sha256=LqS5FBxs6lLLcTkcgxIoLb6OinxShHXR5y4CWZpwmwg,2005
 castor_extractor/utils/client/api/utils.py,sha256=jr8CWf48cIp8QP1P7oZ1zg9WaGlDO3mqCWgQKdEcpyc,238
 castor_extractor/utils/client/api/utils_test.py,sha256=a5aL-pCwa74C8Ne7OT169Bjp8WPDV5Fl8MxNxAllHJg,514
@@ -144,7 +145,7 @@ castor_extractor/visualization/looker/__init__.py,sha256=mem0020YeP4_5zDnqRXOW3g
 castor_extractor/visualization/looker/api/__init__.py,sha256=HDLsLy3kDWHIplAzLl1_u_bvGlgY6cuplf8myJTdfTg,169
 castor_extractor/visualization/looker/api/client.py,sha256=xp7wV59UsrXQXrkR-vB9YH78aPu2rAfwNRFxXegJluo,11283
 castor_extractor/visualization/looker/api/client_test.py,sha256=a80DpBOorFumXEA3D_qHuRZJqR51-DUtbz65XSLuSHc,1977
-castor_extractor/visualization/looker/api/constants.py,sha256=pZpq09tqcGi2Vh8orXxn9eil8ewfPUOLKfVuqgV2W-A,4126
+castor_extractor/visualization/looker/api/constants.py,sha256=wnpEtZNbvTKEsLRCSdDUOru2Y6uIFyrBt1e5Hp9T7J4,4021
 castor_extractor/visualization/looker/api/credentials.py,sha256=dnEMW-d-g4N_JJhkXd-CJcnKLA1zBNMbgnELL_-guNI,972
 castor_extractor/visualization/looker/api/extraction_parameters.py,sha256=53tMtYHxlgALWuKr9w-lOE0xHIqKLvIQHl4w5wufjbU,1284
 castor_extractor/visualization/looker/api/sdk.py,sha256=KEhVCpQ__K9yTxSoIDG5y1FFuAhmeHo65pvxh7g90Ts,1600
@@ -192,17 +193,18 @@ castor_extractor/visualization/mode/client/constants.py,sha256=_Si5AF6VnpoSfnNNg
 castor_extractor/visualization/mode/client/credentials.py,sha256=ptIpCCpoNt06yYaWQgl3Xu78_jVMoqsqWAGqQXVFZlo,606
 castor_extractor/visualization/mode/errors.py,sha256=SKpFT2AiLOuWx2VRLyO7jbAiKcGDFXXrsebpNEKtr0E,1495
 castor_extractor/visualization/mode/extract.py,sha256=g_X7k8L8MldFPbuwOrnyNMF3BEH1r-IAAgNmi3KLF-U,1623
-castor_extractor/visualization/powerbi/__init__.py,sha256=VylJP6kw4yd2zGj31V-U9UXdhnPS9MK2Fz7Sd9KTkKI,119
-castor_extractor/visualization/powerbi/assets.py,sha256=SASUjxtoOMag3NAlZfhpCy0sLap7WfENEMaEZuBrw6o,801
-castor_extractor/visualization/powerbi/client/__init__.py,sha256=ewaEKS_shQlBbCpf-12J-bx3aUIpxbFZRJUL4eNOOno,97
-castor_extractor/visualization/powerbi/client/constants.py,sha256=gW2OdZwPvK_VjzNSZpFYKXW4dyYlSu9h4bIhYnIpXB4,2352
-castor_extractor/visualization/powerbi/client/credentials.py,sha256=z2p3KbA095LFV8XCGV6xLdHuPgOy5sQwCIAjkPgl5n8,772
-castor_extractor/visualization/powerbi/client/credentials_test.py,sha256=Zp7upzTUoGSArg-ujP0d9dGx31OMubIuA3S8X5Flv9Y,1016
-castor_extractor/visualization/powerbi/client/rest.py,sha256=YvDa-jnYVtONibRq6ifw26iphLaHZfGLz5qEmvPNnXI,9782
-castor_extractor/visualization/powerbi/client/rest_test.py,sha256=WMd8042r0nbUZECCVVC9JpJuNica2qlQBBbM8QuYfcQ,8528
-castor_extractor/visualization/powerbi/client/utils.py,sha256=0RcoWcKOdvIGH4f3lYDvufmiMo4tr_ABFlITSrvXjTs,541
-castor_extractor/visualization/powerbi/client/utils_test.py,sha256=ULHL2JLrcv0xjW2r7QF_ce2OaGeeSzajkMDywJ8ZdVA,719
-castor_extractor/visualization/powerbi/extract.py,sha256=OPF2QxP44iruQWARmpAx1HSDj7NLadPApIWVl1yRVZI,1101
+castor_extractor/visualization/powerbi/__init__.py,sha256=AJnmfdmm2mGaInWJkUfZxRqrI7dBkTUSebpow05g5zo,135
+castor_extractor/visualization/powerbi/assets.py,sha256=4VtYLgY81yQ3WzOEDipyK4zkS4xrIY9wjJBO1CeLpb4,932
+castor_extractor/visualization/powerbi/client/__init__.py,sha256=8Bzhd9Z0ebVg2gDchXCOPa80Yqlq_9oCjbGi8u1M6J0,93
+castor_extractor/visualization/powerbi/client/authentication.py,sha256=fz0v9qxeADwA1jiS9UzAQN5mA5kmZT53onlcWon2RGw,892
+castor_extractor/visualization/powerbi/client/client.py,sha256=BDV0m00baYJLK4AyAP-TJ7rp2iEOk0ZsrPSIoSviEHI,7188
+castor_extractor/visualization/powerbi/client/client_test.py,sha256=6NtpcKZCxBWyJO3phnVgE70Wmunb6tWsdXikkReJ02E,5539
+castor_extractor/visualization/powerbi/client/constants.py,sha256=88R_aGachNNUZh6OSH2fkDwZtY4KTStzKm_g7HNCqqo,387
+castor_extractor/visualization/powerbi/client/credentials.py,sha256=ueJ6AySVuigwGxIeQ7tGD2nh0UV1PnhKIkCCqFDvGBw,801
+castor_extractor/visualization/powerbi/client/credentials_test.py,sha256=TzFqxsWVQ3sXR_n0bJsexK9Uz7ceXCEPVqDGWTJzW60,993
+castor_extractor/visualization/powerbi/client/endpoints.py,sha256=WG1JRJ4FznUOQaUCWzjBqRnHNxQKv1-b2ZAnQVwQEVg,2058
+castor_extractor/visualization/powerbi/client/pagination.py,sha256=OZMjoDQPRGMoWd9QcKKrPh3aErJR20SHlrTqY_siLkk,755
+castor_extractor/visualization/powerbi/extract.py,sha256=0YCNSeTqcXSBbrl9g5dJUv8oMm7r-NT8tcfB-IdgPo8,1333
 castor_extractor/visualization/qlik/__init__.py,sha256=u6lIfm_WOykBwt6SlaB7C0Dtx37XBliUbM5oWv26gC8,177
 castor_extractor/visualization/qlik/assets.py,sha256=cG3Cqrj2s4inAqfW6dOaxRape2RPiCeGSYjKsRJRLLo,1657
 castor_extractor/visualization/qlik/client/__init__.py,sha256=5O5N9Jrt3d99agFEJ28lKWs2KkDaXK-lZ07IUtLj56M,130
@@ -282,6 +284,15 @@ castor_extractor/visualization/tableau_revamp/client/gql_queries.py,sha256=-V3To
 castor_extractor/visualization/tableau_revamp/client/rest_fields.py,sha256=gx39X1zMfRVpjmFbgvbgbvtlE0QwxOtk8rZFsIqeGRI,978
 castor_extractor/visualization/tableau_revamp/constants.py,sha256=thS935pJyuZkdciM2EFHbIuTqSFYfB3YGCJYJ_Ls294,55
 castor_extractor/visualization/tableau_revamp/extract.py,sha256=BPy38rFjGG6Nh1eDFeCckE4RHaO-bWW2uhXh7wm8mKk,1368
+castor_extractor/visualization/thoughtspot/__init__.py,sha256=NhTGUk5Kdt54oCjHYoAt0cLBmVLys5lFYiRANL6wCmI,150
+castor_extractor/visualization/thoughtspot/assets.py,sha256=lPRvXk0PKybgLv1AcDVxg-ssf4XLTs0biRqLrqC2TzU,196
+castor_extractor/visualization/thoughtspot/client/__init__.py,sha256=svrE2rMxR-OXctjPeAHMEPePlfcra-9KDevTMcHunAA,86
+castor_extractor/visualization/thoughtspot/client/client.py,sha256=EaJ0x87Ci5-XaPF9x7Gko2efuOmvGvVp2ViLkqlmk1I,3698
+castor_extractor/visualization/thoughtspot/client/credentials.py,sha256=fp4YHiZy-dstWiLr5c4kFU9SyPK5rd2nCeh8k5sVRpM,462
+castor_extractor/visualization/thoughtspot/client/endpoints.py,sha256=u3FRkmG6j5OIMEeXWZcgRObP8JeC4EutIJEeitNV44c,330
+castor_extractor/visualization/thoughtspot/client/utils.py,sha256=54pC7t4-haWrJNPu4R7ef5dbd4zvMe3aep6bP61MglM,874
+castor_extractor/visualization/thoughtspot/client/utils_test.py,sha256=-5ZaEYpQSrIp1-Sx-ViQOLPlv2LoOajEs2mE5YNi_tU,1887
+castor_extractor/visualization/thoughtspot/extract.py,sha256=hpKUpwnAeu3_fPrtmAt6UhB04U8EKZgL7gJp0H7KZoM,1334
 castor_extractor/warehouse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/warehouse/abstract/__init__.py,sha256=Fdfa026tgOo64MvzVRLHM_F2G-JmcehrF0mh3dHgb7s,419
 castor_extractor/warehouse/abstract/asset.py,sha256=Qs7T2Iw7KHgWVT2aAoBfCQ8tB143cUZY-DRUSkpgvGU,2689
@@ -364,11 +375,11 @@ castor_extractor/warehouse/redshift/queries/user.sql,sha256=sEXveJAuNvZacvpI6Wfw
 castor_extractor/warehouse/redshift/queries/view_ddl.sql,sha256=Pkyh_QT6d4rhTeyiVcqw6O8CRl7NEhk2p7eM5YIn5kg,719
 castor_extractor/warehouse/redshift/query.py,sha256=0C81rkt2cpkWrJIxxwALDyqr-49vlqQM04y_N6wwStc,540
 castor_extractor/warehouse/salesforce/__init__.py,sha256=NR4aNea5jeE1xYqeZ_29deeN84CkN0_D_Z7CLQdJvFY,137
-castor_extractor/warehouse/salesforce/client.py,sha256=NbbXTi_eX0ge815FgsiWh4uUnvZMOAl9_mXA_e172_0,3281
+castor_extractor/warehouse/salesforce/client.py,sha256=-9WHcQwEMrpGRQ9CN-bsRSR2Tnx9d-f_FtV4ntsf71w,3287
 castor_extractor/warehouse/salesforce/constants.py,sha256=GusduVBCPvwpk_Im6F3bDvXeNQ7hRnCMdIAjIg65RnE,52
 castor_extractor/warehouse/salesforce/extract.py,sha256=GaxkGWhdksDT_rlT24KX8DMpWnhKlhDMAUvBPGalli0,3454
-castor_extractor/warehouse/salesforce/format.py,sha256=eiPM_4i_m3FEg_2jkMYlhaRBg3gTvV-9xQuk8ghJZiM,3289
-castor_extractor/warehouse/salesforce/format_test.py,sha256=6xDtCxNqvLo5JeHCtXyAun62WMzfVaVsvvMGXXfGgmA,2254
+castor_extractor/warehouse/salesforce/format.py,sha256=DlPD4BQax2RmdDDucw1QbDUTUm2N0CzI7Gc9GymNOYA,3370
+castor_extractor/warehouse/salesforce/format_test.py,sha256=3_OzI0GB3YVEw33ldXCcLG5NwIRZziQaCrAawT4_0g0,3266
 castor_extractor/warehouse/salesforce/pagination.py,sha256=m1S9JRNf6Oe-6dDghYUY5wwTzGzKW5H9pE60PCXMha0,920
 castor_extractor/warehouse/salesforce/soql.py,sha256=XB8ohKwHFfC4Xger7Y84DXLW17IJDye_bZ3FL6DCcOI,1188
 castor_extractor/warehouse/snowflake/__init__.py,sha256=TEGXTyxWp4Tr9gIHb-UFVTRKj6YWmrRtqHruiKSZGiY,174
@@ -402,8 +413,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
 castor_extractor/warehouse/sqlserver/query.py,sha256=j_d5-HMnzBouwGfywVZMRSSwbXzPvzDWlFCZmvxcoGQ,539
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.20.0.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.20.0.dist-info/METADATA,sha256=u6XUIwUCR1lLswRR_mMI9sm9zXrqIbE05MGVnAPLOBA,20819
-castor_extractor-0.20.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-castor_extractor-0.20.0.dist-info/entry_points.txt,sha256=X_pDYOmhUUMbiAD9h2GZveuGdT8UgL38KJqP44xkvqo,1495
-castor_extractor-0.20.0.dist-info/RECORD,,
+castor_extractor-0.20.4.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.20.4.dist-info/METADATA,sha256=YcFx5O-gccq_JevTWl9xfeE5LGf5baiUKHfPrG1QX28,21123
+castor_extractor-0.20.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+castor_extractor-0.20.4.dist-info/entry_points.txt,sha256=IVGy_oM8VjzADMAxzmiNJTYYidTCsI98MpO_mkXjkqE,1573
+castor_extractor-0.20.4.dist-info/RECORD,,

{castor_extractor-0.20.0.dist-info → castor_extractor-0.20.4.dist-info}/entry_points.txt

@@ -18,6 +18,7 @@ castor-extract-sigma=castor_extractor.commands.extract_sigma:main
 castor-extract-snowflake=castor_extractor.commands.extract_snowflake:main
 castor-extract-sqlserver=castor_extractor.commands.extract_sqlserver:main
 castor-extract-tableau=castor_extractor.commands.extract_tableau:main
+castor-extract-thoughtspot=castor_extractor.commands.extract_thoughtspot:main
 castor-file-check=castor_extractor.commands.file_check:main
 castor-upload=castor_extractor.commands.upload:main
 

castor_extractor/visualization/powerbi/client/rest.py

@@ -1,305 +0,0 @@
-import logging
-from datetime import date, datetime
-from time import sleep
-from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
-
-import msal  # type: ignore
-import requests
-
-from ....utils import at_midnight, format_date, yesterday
-from ..assets import PowerBiAsset
-from .constants import (
-    DEFAULT_TIMEOUT_IN_SECS,
-    GET,
-    POST,
-    SCAN_READY,
-    Batches,
-    Keys,
-    QueryParams,
-    Urls,
-)
-from .credentials import PowerbiCredentials
-from .utils import batch_size_is_valid_or_assert, datetime_is_recent_or_assert
-
-logger = logging.getLogger(__name__)
-
-
-def _time_filter(day: Optional[date]) -> Tuple[datetime, datetime]:
-    target_day = day or yesterday()
-    start = at_midnight(target_day)
-    end = datetime.combine(target_day, datetime.max.time())
-    return start, end
-
-
-def _url(
-    day: Optional[date],
-    continuation_uri: Optional[str],
-) -> str:
-    if continuation_uri:
-        return continuation_uri
-
-    url = Urls.ACTIVITY_EVENTS
-    start, end = _time_filter(day)
-    url += "?$filter=Activity eq 'viewreport'"
-    url += f"&startDateTime='{format_date(start)}'"
-    url += f"&endDateTime='{format_date(end)}'"
-    return url
-
-
-class Client:
-    """
-    PowerBI rest admin api
-    https://learn.microsoft.com/en-us/rest/api/power-bi/admin
-    """
-
-    def __init__(self, credentials: PowerbiCredentials):
-        self.creds = credentials
-        client_app = f"{Urls.CLIENT_APP_BASE}{self.creds.tenant_id}"
-        self.app = msal.ConfidentialClientApplication(
-            client_id=self.creds.client_id,
-            authority=client_app,
-            client_credential=self.creds.secret,
-        )
-
-    def _access_token(self) -> dict:
-        token = self.app.acquire_token_for_client(scopes=self.creds.scopes)
-
-        if Keys.ACCESS_TOKEN not in token:
-            raise ValueError(f"No access token in token response: {token}")
-
-        return token
-
-    def _header(self) -> Dict:
-        """Return header used in following rest api call"""
-        token = self._access_token()
-        return {"Authorization": f"Bearer {token[Keys.ACCESS_TOKEN]}"}
-
-    def _call(
-        self,
-        url: str,
-        method: str = GET,
-        *,
-        params: Optional[Dict] = None,
-        data: Optional[dict] = None,
-        processor: Optional[Callable] = None,
-    ) -> Any:
-        """
-        Make either a get or a post http request.Request, by default
-        result.json is returned. Optionally you can provide a processor callback
-        to transform the result.
-        """
-        logger.debug(f"Calling {method} on {url}")
-        result = requests.request(
-            method,
-            url,
-            headers=self._header(),
-            params=params,
-            data=data,
-        )
-        result.raise_for_status()
-
-        if processor:
-            return processor(result)
-
-        return result.json()
-
-    def _get(
-        self,
-        url: str,
-        *,
-        params: Optional[Dict] = None,
-        processor: Optional[Callable] = None,
-    ) -> Any:
-        return self._call(url, GET, params=params, processor=processor)
-
-    def _post(
-        self,
-        url: str,
-        *,
-        params: Optional[dict],
-        data: Optional[dict],
-        processor: Optional[Callable] = None,
-    ) -> Any:
-        return self._call(
-            url,
-            POST,
-            params=params,
-            data=data,
-            processor=processor,
-        )
-
-    def _workspace_ids(
-        self,
-        modified_since: Optional[datetime] = None,
-    ) -> List[str]:
-        """
-        Get workspaces ids from powerBI admin API.
-        If modified_since, take only workspaces that have been modified since
-
-        more: https://learn.microsoft.com/en-us/rest/api/power-bi/admin/workspace-info-get-modified-workspaces
-        """
-
-        def result_callback(call_result: requests.models.Response) -> List[str]:
-            return [x["id"] for x in call_result.json()]
-
-        params: Dict[str, Union[bool, str]] = {
-            Keys.INACTIVE_WORKSPACES: True,
-            Keys.PERSONAL_WORKSPACES: True,
-        }
-
-        if modified_since:
-            datetime_is_recent_or_assert(modified_since)
-            modified_since_iso = f"{modified_since.isoformat()}0Z"
-            params[Keys.MODIFIED_SINCE] = modified_since_iso
-
-        result = self._get(
-            Urls.WORKSPACE_IDS,
-            params=params,
-            processor=result_callback,
-        )
-
-        return result
-
-    def _create_scan(self, workspaces_ids: List[str]) -> int:
-        batch_size_is_valid_or_assert(workspaces_ids)
-        request_body = {"workspaces": workspaces_ids}
-        params = QueryParams.METADATA_SCAN
-        scan_id = self._post(
-            Urls.METADATA_POST,
-            params=params,
-            data=request_body,
-        )
-        return scan_id[Keys.ID]
-
-    def _wait_for_scan_result(self, scan_id: int) -> bool:
-        url = f"{Urls.METADATA_WAIT}/{scan_id}"
-        waiting_seconds = 0
-        sleep_seconds = 1
-        while True:
-            result = self._get(url, processor=lambda x: x)
-            if result.status_code != 200:
-                return False
-            if result.json()[Keys.STATUS] == SCAN_READY:
-                logger.info(f"scan {scan_id} ready")
-                return True
-            if waiting_seconds >= DEFAULT_TIMEOUT_IN_SECS:
-                break
-            waiting_seconds += sleep_seconds
-            logger.info(
-                f"Waiting {sleep_seconds} sec for scan {scan_id} to be ready…",
-            )
-            sleep(sleep_seconds)
-        return False
-
-    def _get_scan(self, scan_id: int) -> List[dict]:
-        url = f"{Urls.METADATA_GET}/{scan_id}"
-        return self._get(url)[Keys.WORKSPACES]
-
-    def _activity_events(
-        self,
-        *,
-        day: Optional[date] = None,
-        continuation_uri: Optional[str] = None,
-    ) -> List[Dict]:
-        """
-        Returns a list of activity events for the organization.
-        https://learn.microsoft.com/en-us/power-bi/admin/service-admin-auditing#activityevents-rest-api
-        - when no day is specified, fallback is yesterday
-        - continuation_uri allows to fetch paginated data (internal usage)
-        """
-        url = _url(day, continuation_uri)
-        answer = self._get(url)
-        activity_events = answer[Keys.ACTIVITY_EVENT_ENTITIES]
-        is_last = answer[Keys.LAST_RESULT_SET]
-        assert isinstance(is_last, bool)
-        if is_last:
-            return activity_events
-
-        # there are more data to fetch
-        # https://learn.microsoft.com/en-us/rest/api/power-bi/admin/get-activity-events#get-the-next-set-of-audit-activity-events-by-sending-the-continuation-token-to-the-api-example
-        continuation_uri = answer[Keys.CONTINUATION_URI]
-        rest = self._activity_events(continuation_uri=continuation_uri)
-        activity_events.extend(rest)
-        return activity_events
-
-    def _datasets(self) -> List[Dict]:
-        """
-        Returns a list of datasets for the organization.
-        https://learn.microsoft.com/en-us/rest/api/power-bi/admin/datasets-get-datasets-as-admin
-        """
-        return self._get(Urls.DATASETS)[Keys.VALUE]
-
-    def _reports(self) -> List[Dict]:
-        """
-        Returns a list of reports for the organization.
-        https://learn.microsoft.com/en-us/rest/api/power-bi/admin/reports-get-reports-as-admin
-        """
-        reports = self._get(Urls.REPORTS)[Keys.VALUE]
-        for report in reports:
-            report_id = report.get("id")
-            try:
-                url = Urls.REPORTS + f"/{report_id}/pages"
-                pages = self._get(url)[Keys.VALUE]
-                report["pages"] = pages
-            except (requests.HTTPError, requests.exceptions.Timeout) as e:
-                logger.debug(e)
-                continue
-        return reports
-
-    def _dashboards(self) -> List[Dict]:
-        """
-        Returns a list of dashboards for the organization.
-        https://learn.microsoft.com/en-us/rest/api/power-bi/admin/dashboards-get-dashboards-as-admin
-        """
-        return self._get(Urls.DASHBOARD)[Keys.VALUE]
-
-    def _metadata(
-        self,
-        modified_since: Optional[datetime] = None,
-    ) -> Iterator[List[Dict]]:
-        """
-        Fetch metadata by workspace.
-        https://learn.microsoft.com/en-us/power-bi/enterprise/service-admin-metadata-scanning
-        """
-        ids = self._workspace_ids(modified_since)
-
-        for ix in range(0, len(ids), Batches.METADATA):
-            batch_ids = [w_id for w_id in ids[ix : ix + Batches.METADATA]]
-            scan_id = self._create_scan(batch_ids)
-            self._wait_for_scan_result(scan_id)
-            yield self._get_scan(scan_id)
-
-    def test_connection(self) -> None:
-        """Use credentials & verify requesting the API doesn't raise an error"""
-        self._header()
-
-    def fetch(
-        self,
-        asset: PowerBiAsset,
-        *,
-        modified_since: Optional[datetime] = None,
-        day: Optional[date] = None,
-    ) -> List[Dict]:
-        """
-        Given a PowerBi asset, returns the corresponding data using the
-        appropriate client.
-        """
-        logger.info(f"Starting extraction of {asset}")
-        asset = PowerBiAsset(asset)
-
-        if asset == PowerBiAsset.ACTIVITY_EVENTS:
-            return self._activity_events(day=day)
-
-        if asset == PowerBiAsset.DATASETS:
-            return self._datasets()
-
-        if asset == PowerBiAsset.DASHBOARDS:
-            return self._dashboards()
-
-        if asset == PowerBiAsset.REPORTS:
-            return self._reports()
-
-        assert asset == PowerBiAsset.METADATA
-        return [
-            item for batch in self._metadata(modified_since) for item in batch
-        ]