castor-extractor 0.23.3__py3-none-any.whl → 0.24.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of castor-extractor might be problematic. Click here for more details.
- CHANGELOG.md +4 -0
- castor_extractor/knowledge/confluence/client/client.py +0 -13
- castor_extractor/knowledge/confluence/client/endpoints.py +1 -1
- {castor_extractor-0.23.3.dist-info → castor_extractor-0.24.0.dist-info}/METADATA +5 -1
- {castor_extractor-0.23.3.dist-info → castor_extractor-0.24.0.dist-info}/RECORD +8 -43
- castor_extractor/visualization/tableau/__init__.py +0 -3
- castor_extractor/visualization/tableau/assets.py +0 -49
- castor_extractor/visualization/tableau/client/__init__.py +0 -2
- castor_extractor/visualization/tableau/client/client.py +0 -229
- castor_extractor/visualization/tableau/client/client_utils.py +0 -75
- castor_extractor/visualization/tableau/client/credentials.py +0 -104
- castor_extractor/visualization/tableau/client/project.py +0 -28
- castor_extractor/visualization/tableau/client/safe_mode.py +0 -70
- castor_extractor/visualization/tableau/constants.py +0 -9
- castor_extractor/visualization/tableau/errors.py +0 -5
- castor_extractor/visualization/tableau/extract.py +0 -121
- castor_extractor/visualization/tableau/gql_fields.py +0 -249
- castor_extractor/visualization/tableau/tests/__init__.py +0 -0
- castor_extractor/visualization/tableau/tests/unit/__init__.py +0 -0
- castor_extractor/visualization/tableau/tests/unit/assets/graphql/metadata/metadata_1_get.json +0 -15
- castor_extractor/visualization/tableau/tests/unit/assets/graphql/metadata/metadata_2_get.json +0 -15
- castor_extractor/visualization/tableau/tests/unit/assets/rest_api/auth.xml +0 -7
- castor_extractor/visualization/tableau/tests/unit/assets/rest_api/project_get.xml +0 -9
- castor_extractor/visualization/tableau/tests/unit/assets/rest_api/user_get.xml +0 -8
- castor_extractor/visualization/tableau/tests/unit/assets/rest_api/view_get_usage.xml +0 -24
- castor_extractor/visualization/tableau/tests/unit/assets/rest_api/workbook_get.xml +0 -19
- castor_extractor/visualization/tableau/tests/unit/graphql/__init__.py +0 -0
- castor_extractor/visualization/tableau/tests/unit/graphql/paginated_object_test.py +0 -63
- castor_extractor/visualization/tableau/tests/unit/rest_api/__init__.py +0 -0
- castor_extractor/visualization/tableau/tests/unit/rest_api/auth_test.py +0 -39
- castor_extractor/visualization/tableau/tests/unit/rest_api/credentials_test.py +0 -13
- castor_extractor/visualization/tableau/tests/unit/rest_api/projects_test.py +0 -59
- castor_extractor/visualization/tableau/tests/unit/rest_api/usages_test.py +0 -49
- castor_extractor/visualization/tableau/tests/unit/rest_api/users_test.py +0 -52
- castor_extractor/visualization/tableau/tests/unit/rest_api/workbooks_test.py +0 -60
- castor_extractor/visualization/tableau/tests/unit/utils/__init__.py +0 -1
- castor_extractor/visualization/tableau/tests/unit/utils/env_key.py +0 -6
- castor_extractor/visualization/tableau/tsc_fields.py +0 -46
- castor_extractor/visualization/tableau/types.py +0 -11
- castor_extractor/visualization/tableau/usage.py +0 -14
- {castor_extractor-0.23.3.dist-info → castor_extractor-0.24.0.dist-info}/LICENCE +0 -0
- {castor_extractor-0.23.3.dist-info → castor_extractor-0.24.0.dist-info}/WHEEL +0 -0
- {castor_extractor-0.23.3.dist-info → castor_extractor-0.24.0.dist-info}/entry_points.txt +0 -0
CHANGELOG.md
CHANGED
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
from collections.abc import Iterator
|
|
2
2
|
from functools import partial
|
|
3
|
-
from http import HTTPStatus
|
|
4
|
-
from typing import Optional
|
|
5
3
|
|
|
6
4
|
from ....utils import (
|
|
7
5
|
APIClient,
|
|
8
6
|
BasicAuth,
|
|
9
|
-
RequestSafeMode,
|
|
10
7
|
fetch_all_pages,
|
|
11
8
|
)
|
|
12
9
|
from ..assets import (
|
|
@@ -21,19 +18,11 @@ _HEADERS = {
|
|
|
21
18
|
"Content-Type": "application/json",
|
|
22
19
|
}
|
|
23
20
|
|
|
24
|
-
_MAX_ERROR_IGNORED_COUNT = 10
|
|
25
|
-
_IGNORED_ERROR_CODES = (HTTPStatus.BAD_GATEWAY,)
|
|
26
|
-
_SAFE_MODE = RequestSafeMode(
|
|
27
|
-
max_errors=_MAX_ERROR_IGNORED_COUNT,
|
|
28
|
-
status_codes=_IGNORED_ERROR_CODES,
|
|
29
|
-
)
|
|
30
|
-
|
|
31
21
|
|
|
32
22
|
class ConfluenceClient(APIClient):
|
|
33
23
|
def __init__(
|
|
34
24
|
self,
|
|
35
25
|
credentials: ConfluenceCredentials,
|
|
36
|
-
safe_mode: Optional[RequestSafeMode] = None,
|
|
37
26
|
):
|
|
38
27
|
self.account_id = credentials.account_id
|
|
39
28
|
auth = BasicAuth(
|
|
@@ -43,14 +32,12 @@ class ConfluenceClient(APIClient):
|
|
|
43
32
|
auth=auth,
|
|
44
33
|
host=credentials.base_url,
|
|
45
34
|
headers=_HEADERS,
|
|
46
|
-
safe_mode=safe_mode or _SAFE_MODE,
|
|
47
35
|
)
|
|
48
36
|
|
|
49
37
|
def pages(self):
|
|
50
38
|
request = partial(
|
|
51
39
|
self._get,
|
|
52
40
|
endpoint=ConfluenceEndpointFactory.pages(),
|
|
53
|
-
params={"body-format": "atlas_doc_format"},
|
|
54
41
|
)
|
|
55
42
|
yield from fetch_all_pages(request, ConfluencePagination)
|
|
56
43
|
|
|
@@ -14,7 +14,7 @@ class ConfluenceEndpointFactory:
|
|
|
14
14
|
Endpoint to fetch all pages.
|
|
15
15
|
More: https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-page/#api-pages-get
|
|
16
16
|
"""
|
|
17
|
-
return f"{cls.API}{cls.PAGES}"
|
|
17
|
+
return f"{cls.API}{cls.PAGES}?body-format=atlas_doc_format"
|
|
18
18
|
|
|
19
19
|
@classmethod
|
|
20
20
|
def users(cls) -> str:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: castor-extractor
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.24.0
|
|
4
4
|
Summary: Extract your metadata assets.
|
|
5
5
|
Home-page: https://www.castordoc.com/
|
|
6
6
|
License: EULA
|
|
@@ -208,6 +208,10 @@ For any questions or bug report, contact us at [support@castordoc.com](mailto:su
|
|
|
208
208
|
|
|
209
209
|
# Changelog
|
|
210
210
|
|
|
211
|
+
## 0.24.0 - 2025-03-10
|
|
212
|
+
|
|
213
|
+
* Remove legacy Tableau Connector
|
|
214
|
+
|
|
211
215
|
## 0.23.3 - 2025-02-19
|
|
212
216
|
|
|
213
217
|
* Snowflake : add --insecure-mode option to turn off OCSP checking
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
CHANGELOG.md,sha256=
|
|
1
|
+
CHANGELOG.md,sha256=Ud6kiuDEyG_gu0YAVlp9-oXxriLibyma98FDrlajJJE,15756
|
|
2
2
|
Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
|
|
3
3
|
DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
|
|
4
4
|
LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
@@ -43,9 +43,9 @@ castor_extractor/knowledge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
|
|
|
43
43
|
castor_extractor/knowledge/confluence/__init__.py,sha256=pRT615pMDlB7Ifs09erVn2EdpZHgkvX5selemWU3VPE,129
|
|
44
44
|
castor_extractor/knowledge/confluence/assets.py,sha256=zv2G2LB8H0fKDbVJ4kHrAjbqehXI_K-wgd_ghSXGFvs,144
|
|
45
45
|
castor_extractor/knowledge/confluence/client/__init__.py,sha256=ALAzo0JEhxFzH2FnIO6HmtkAGS2_bGY8KXXMcTGV3aE,84
|
|
46
|
-
castor_extractor/knowledge/confluence/client/client.py,sha256=
|
|
46
|
+
castor_extractor/knowledge/confluence/client/client.py,sha256=F8A_ckZ4ojJC8BnAXeAIHUC2oOQMBWnTfqQwJbAyTns,1689
|
|
47
47
|
castor_extractor/knowledge/confluence/client/credentials.py,sha256=tqUMw-SVoAi4o6I6OeGk4MeDiIPU3-ihhaomXv4CQ64,419
|
|
48
|
-
castor_extractor/knowledge/confluence/client/endpoints.py,sha256=
|
|
48
|
+
castor_extractor/knowledge/confluence/client/endpoints.py,sha256=eWMKtjDUPGoKR8Nqq18JTIoEq913GNo1Klm9RduIOHM,765
|
|
49
49
|
castor_extractor/knowledge/confluence/client/pagination.py,sha256=ty4meiMEujDVSiQyOJTibd-ReYyDyGezdFuk7EAGtMA,862
|
|
50
50
|
castor_extractor/knowledge/confluence/extract.py,sha256=6pA68CmYNC50qCJ7NgZMW0jD4ev0a_ltI5kSyBqSC0U,1565
|
|
51
51
|
castor_extractor/knowledge/notion/__init__.py,sha256=ZDmh0eNSxHf1zVPm0aYlKPci-vzOXhAgdsWjS2hdjh4,117
|
|
@@ -257,41 +257,6 @@ castor_extractor/visualization/sigma/client/credentials.py,sha256=XddAuQSmCKpxJ7
|
|
|
257
257
|
castor_extractor/visualization/sigma/client/endpoints.py,sha256=DBFphbgoH78_MZUGM_bKBAq28Nl7LWSZ6VRsbxrxtDg,1162
|
|
258
258
|
castor_extractor/visualization/sigma/client/pagination.py,sha256=kNEhNq08tTGbypyMjxs0w4uvDtQc_iaWpOZweaa_FsU,690
|
|
259
259
|
castor_extractor/visualization/sigma/extract.py,sha256=XIT1qsj6g6dgBWP8HPfj_medZexu48EaY9tUwi14gzM,2298
|
|
260
|
-
castor_extractor/visualization/tableau/__init__.py,sha256=hDohrWjkorrX01JMc154aa9vi3ZqBKmA1lkfQtMFfYE,114
|
|
261
|
-
castor_extractor/visualization/tableau/assets.py,sha256=jid6bPym-h5uMP5NwNVSpsYEn2KOr24tzSjtVuH5o-k,1247
|
|
262
|
-
castor_extractor/visualization/tableau/client/__init__.py,sha256=FQX1MdxS8Opn3Oyq8eby7suk3ANbLlpzzCPQ3zqvk0I,78
|
|
263
|
-
castor_extractor/visualization/tableau/client/client.py,sha256=0adwnNumC37BmtWsqhRsj8XHkvo-Zc0Eymp5QAuPSP0,7357
|
|
264
|
-
castor_extractor/visualization/tableau/client/client_utils.py,sha256=s5jQhCozc1AMldTx-obi7K1Kxm2vS9cYolhc0fGBAB4,2115
|
|
265
|
-
castor_extractor/visualization/tableau/client/credentials.py,sha256=kvj8B9fB5DlcVczKEtB3CYO2a82vvwsG4KEjjUBx3tM,3380
|
|
266
|
-
castor_extractor/visualization/tableau/client/project.py,sha256=uLlZ5-eZI_4VxBmEB5d1gWy_X_w6uVt2EKoiX9cJ0UA,812
|
|
267
|
-
castor_extractor/visualization/tableau/client/safe_mode.py,sha256=jNmEQFcpoJfOWkFEsEk8Gimj7wIeXH3_OXiulwhKD1s,2015
|
|
268
|
-
castor_extractor/visualization/tableau/constants.py,sha256=O2CqeviFz122BumNHoJ1N-e1lzyqIHF9OYnGQttg4hg,126
|
|
269
|
-
castor_extractor/visualization/tableau/errors.py,sha256=WWvmnp5pdxFJqanPKeDRADZc0URSPxkJqxDI6bwoifQ,91
|
|
270
|
-
castor_extractor/visualization/tableau/extract.py,sha256=C32FmEhfFOMvEx1hCgbnOYn_LBiORQwqEt7JH73F1k8,3306
|
|
271
|
-
castor_extractor/visualization/tableau/gql_fields.py,sha256=nwG2VAOI1AX75i4sWirNsjEB82hBWeHF1Hce5b2Mz5A,5520
|
|
272
|
-
castor_extractor/visualization/tableau/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
273
|
-
castor_extractor/visualization/tableau/tests/unit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
274
|
-
castor_extractor/visualization/tableau/tests/unit/assets/graphql/metadata/metadata_1_get.json,sha256=4iMvJ_VakDa67xN2ROraAccaz_DDxX6Y5Y1XnTU5F5Y,446
|
|
275
|
-
castor_extractor/visualization/tableau/tests/unit/assets/graphql/metadata/metadata_2_get.json,sha256=ZpkJorxFmk9CJRjZReI3qGkoZFKa1GHD3JOxrDgdOrY,447
|
|
276
|
-
castor_extractor/visualization/tableau/tests/unit/assets/rest_api/auth.xml,sha256=gAuOdNzLhzqj9EdeCHMLq9tf9-BOe0kC4OjW1cMePeo,450
|
|
277
|
-
castor_extractor/visualization/tableau/tests/unit/assets/rest_api/project_get.xml,sha256=Eg2Pvp2VZ3O9rtUZzhak3fEODS007AFIc_konEQGRdg,1018
|
|
278
|
-
castor_extractor/visualization/tableau/tests/unit/assets/rest_api/user_get.xml,sha256=q0dT6jiSdTj-T50VD7hlv3cf-acSIK_OlQulKDyZ39U,679
|
|
279
|
-
castor_extractor/visualization/tableau/tests/unit/assets/rest_api/view_get_usage.xml,sha256=UHfPPPETt0rrXde6EjsNdrOGdbVqIyeQwDwRU0nP5fY,1415
|
|
280
|
-
castor_extractor/visualization/tableau/tests/unit/assets/rest_api/workbook_get.xml,sha256=MLyS2AbnPZxkAClCxOK6_DTVoiK4TQjzcw--lQ2Jug8,1325
|
|
281
|
-
castor_extractor/visualization/tableau/tests/unit/graphql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
282
|
-
castor_extractor/visualization/tableau/tests/unit/graphql/paginated_object_test.py,sha256=stFLqx8ifMBe75D62Bq6izn26kLgvxrmyBxGZH2wK4c,1922
|
|
283
|
-
castor_extractor/visualization/tableau/tests/unit/rest_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
284
|
-
castor_extractor/visualization/tableau/tests/unit/rest_api/auth_test.py,sha256=L3LfrD9DGISdg0YZL9Ma0ZE_ab2jWfRNLL_mqrh2vM4,1123
|
|
285
|
-
castor_extractor/visualization/tableau/tests/unit/rest_api/credentials_test.py,sha256=IXemuC0wO_BYuU-wm_O-NUfm92RnIJeu98sogA3pmf4,331
|
|
286
|
-
castor_extractor/visualization/tableau/tests/unit/rest_api/projects_test.py,sha256=ySm56VP4Iy7CZYHUEdrBow9_YKGBc7XwHbvzZRJEKUo,1779
|
|
287
|
-
castor_extractor/visualization/tableau/tests/unit/rest_api/usages_test.py,sha256=OdY9OehUEjSsHnOJ7lR906oEvIo5k2RV9y-Zvgy_XMs,1401
|
|
288
|
-
castor_extractor/visualization/tableau/tests/unit/rest_api/users_test.py,sha256=-D3mZtnsXQd5xUow8nYo5Ru2Rl9mXDdGGQZ5bVzAmJg,1499
|
|
289
|
-
castor_extractor/visualization/tableau/tests/unit/rest_api/workbooks_test.py,sha256=ga5MK2aN0BNw-Vo7g8OJk8ORmIgrIE8VIXG6p5CO748,1979
|
|
290
|
-
castor_extractor/visualization/tableau/tests/unit/utils/__init__.py,sha256=IzeQqv7EBwGIBh50y6TEsUVXXpqXLjaeBkZeAb0CisE,26
|
|
291
|
-
castor_extractor/visualization/tableau/tests/unit/utils/env_key.py,sha256=fBX8pG1zOJmoUwCTFtDMVinRVlnFDbMI3wBBRNVr_GM,203
|
|
292
|
-
castor_extractor/visualization/tableau/tsc_fields.py,sha256=zMlCMCop3ni69So5DjvxKWEG15Kwh-x8gaSwErSVRGY,937
|
|
293
|
-
castor_extractor/visualization/tableau/types.py,sha256=EvyTCsBs33Xp3RIJg5MXhbtEF_G2-T2CE-oVDjOBA_U,303
|
|
294
|
-
castor_extractor/visualization/tableau/usage.py,sha256=LlFwlbEr-EnYUJjKZha99CRCRrERJ350oAvzBQlp9_s,427
|
|
295
260
|
castor_extractor/visualization/tableau_revamp/__init__.py,sha256=a3DGjQhaz17gBqW-E84TAgupKbqLC40y5Ajo1yn-ot4,156
|
|
296
261
|
castor_extractor/visualization/tableau_revamp/assets.py,sha256=8sJsK6Qixao6xVmVaO1usvs16SjNub9sIx7o-adYV14,659
|
|
297
262
|
castor_extractor/visualization/tableau_revamp/client/__init__.py,sha256=wmS9uLtUiqNYVloi0-DgD8d2qzu3RVZEAtWiaDp6G_M,90
|
|
@@ -436,8 +401,8 @@ castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=kbBQP-TdG5px1IVgyx
|
|
|
436
401
|
castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=gOrZsMVypusR2dc4vwVs4E1a-CliRsr_UjnD2EbXs-A,94
|
|
437
402
|
castor_extractor/warehouse/sqlserver/query.py,sha256=g0hPT-RmeGi2DyenAi3o72cTlQsLToXIFYojqc8E5fQ,533
|
|
438
403
|
castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
|
|
439
|
-
castor_extractor-0.
|
|
440
|
-
castor_extractor-0.
|
|
441
|
-
castor_extractor-0.
|
|
442
|
-
castor_extractor-0.
|
|
443
|
-
castor_extractor-0.
|
|
404
|
+
castor_extractor-0.24.0.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
|
|
405
|
+
castor_extractor-0.24.0.dist-info/METADATA,sha256=HWTKSDx_akRg3FK_dMP5mRxlLg9Oc55uGUYgt2NmYaQ,22828
|
|
406
|
+
castor_extractor-0.24.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
407
|
+
castor_extractor-0.24.0.dist-info/entry_points.txt,sha256=7aVSxc-_2dicp28Ow-S4y0p4wGoTm9zGmVptMvfLdw8,1649
|
|
408
|
+
castor_extractor-0.24.0.dist-info/RECORD,,
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
|
-
|
|
3
|
-
from ...types import ExternalAsset, classproperty
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class TableauAsset(ExternalAsset):
|
|
7
|
-
"""
|
|
8
|
-
Tableau assets
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
CUSTOM_SQL_TABLE = "custom_sql_tables"
|
|
12
|
-
CUSTOM_SQL_QUERY = "custom_sql_queries"
|
|
13
|
-
DASHBOARD = "dashboards"
|
|
14
|
-
DASHBOARD_SHEET = "dashboards_sheets"
|
|
15
|
-
DATASOURCE = "datasources"
|
|
16
|
-
FIELD = "fields"
|
|
17
|
-
PROJECT = "projects"
|
|
18
|
-
PUBLISHED_DATASOURCE = "published_datasources"
|
|
19
|
-
SHEET = "sheets"
|
|
20
|
-
USAGE = "views"
|
|
21
|
-
USER = "users"
|
|
22
|
-
WORKBOOK = "workbooks"
|
|
23
|
-
WORKBOOK_TO_DATASOURCE = "workbooks_to_datasource"
|
|
24
|
-
|
|
25
|
-
@classproperty
|
|
26
|
-
def optional(cls) -> set["TableauAsset"]:
|
|
27
|
-
return {
|
|
28
|
-
TableauAsset.DASHBOARD,
|
|
29
|
-
TableauAsset.DASHBOARD_SHEET,
|
|
30
|
-
TableauAsset.FIELD,
|
|
31
|
-
TableauAsset.SHEET,
|
|
32
|
-
TableauAsset.PUBLISHED_DATASOURCE,
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class TableauGraphqlAsset(Enum):
|
|
37
|
-
"""
|
|
38
|
-
Assets which can be fetched from Tableau
|
|
39
|
-
"""
|
|
40
|
-
|
|
41
|
-
BIN_FIELD = "binFields"
|
|
42
|
-
CALCULATED_FIELD = "calculatedFields"
|
|
43
|
-
COLUMN_FIELD = "columnFields"
|
|
44
|
-
CUSTOM_SQL = "customSQLTables"
|
|
45
|
-
DASHBOARD = "dashboards"
|
|
46
|
-
DATASOURCE = "datasources"
|
|
47
|
-
GROUP_FIELD = "groupFields"
|
|
48
|
-
SHEETS = "sheets"
|
|
49
|
-
WORKBOOK_TO_DATASOURCE = "workbooks"
|
|
@@ -1,229 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
|
|
3
|
-
import tableauserverclient as TSC # type: ignore
|
|
4
|
-
|
|
5
|
-
from ....utils import SerializedAsset
|
|
6
|
-
from ..assets import TableauAsset
|
|
7
|
-
from ..constants import PAGE_SIZE, TABLEAU_SERVER_VERSION
|
|
8
|
-
from ..usage import compute_usage_views
|
|
9
|
-
from .client_utils import extract_asset, get_paginated_objects
|
|
10
|
-
from .credentials import CredentialsApi, CredentialsKey, get_value
|
|
11
|
-
from .project import compute_project_path
|
|
12
|
-
from .safe_mode import safe_mode_fetch_usage
|
|
13
|
-
|
|
14
|
-
logger = logging.getLogger(__name__)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class ApiClient:
|
|
18
|
-
"""
|
|
19
|
-
Connect to Tableau REST API and fetch main assets.
|
|
20
|
-
Superuser credentials are required.
|
|
21
|
-
https://tableau.github.io/server-client-python/docs/
|
|
22
|
-
"""
|
|
23
|
-
|
|
24
|
-
def __init__(
|
|
25
|
-
self,
|
|
26
|
-
**kwargs,
|
|
27
|
-
):
|
|
28
|
-
self._credentials = CredentialsApi(
|
|
29
|
-
user=get_value(CredentialsKey.TABLEAU_USER, kwargs, True),
|
|
30
|
-
password=get_value(CredentialsKey.TABLEAU_PASSWORD, kwargs, True),
|
|
31
|
-
token_name=get_value(
|
|
32
|
-
CredentialsKey.TABLEAU_TOKEN_NAME,
|
|
33
|
-
kwargs,
|
|
34
|
-
True,
|
|
35
|
-
),
|
|
36
|
-
token=get_value(CredentialsKey.TABLEAU_TOKEN, kwargs, True),
|
|
37
|
-
server_url=get_value(CredentialsKey.TABLEAU_SERVER_URL, kwargs),
|
|
38
|
-
site_id=get_value(CredentialsKey.TABLEAU_SITE_ID, kwargs, True),
|
|
39
|
-
)
|
|
40
|
-
self._server = TSC.Server(self._credentials.server_url)
|
|
41
|
-
self._server.add_http_options({"verify": True})
|
|
42
|
-
self._page_size = PAGE_SIZE
|
|
43
|
-
self._server.version = TABLEAU_SERVER_VERSION
|
|
44
|
-
self._safe_mode = bool(kwargs.get("safe_mode"))
|
|
45
|
-
self.errors: list[str] = []
|
|
46
|
-
|
|
47
|
-
@staticmethod
|
|
48
|
-
def name() -> str:
|
|
49
|
-
return "Tableau/API"
|
|
50
|
-
|
|
51
|
-
def _user_password_login(self) -> None:
|
|
52
|
-
"""Login into Tableau using user and password"""
|
|
53
|
-
self._server.auth.sign_in(
|
|
54
|
-
TSC.TableauAuth(
|
|
55
|
-
self._credentials.user,
|
|
56
|
-
self._credentials.password,
|
|
57
|
-
site_id=self._credentials.site_id,
|
|
58
|
-
),
|
|
59
|
-
)
|
|
60
|
-
|
|
61
|
-
def _pat_login(self) -> None:
|
|
62
|
-
"""Login into Tableau using personal authentication token"""
|
|
63
|
-
self._server.auth.sign_in(
|
|
64
|
-
TSC.PersonalAccessTokenAuth(
|
|
65
|
-
self._credentials.token_name,
|
|
66
|
-
self._credentials.token,
|
|
67
|
-
site_id=self._credentials.site_id,
|
|
68
|
-
),
|
|
69
|
-
)
|
|
70
|
-
|
|
71
|
-
def login(self) -> None:
|
|
72
|
-
"""Login into Tableau"""
|
|
73
|
-
|
|
74
|
-
if self._credentials.user and self._credentials.password:
|
|
75
|
-
logger.info("Logging in using user and password authentication")
|
|
76
|
-
return self._user_password_login()
|
|
77
|
-
|
|
78
|
-
if self._credentials.token_name and self._credentials.token:
|
|
79
|
-
logger.info("Logging in using token authentication")
|
|
80
|
-
return self._pat_login()
|
|
81
|
-
|
|
82
|
-
raise ValueError(
|
|
83
|
-
"""Wrong authentication: you should provide either user and password
|
|
84
|
-
or personal access token""",
|
|
85
|
-
)
|
|
86
|
-
|
|
87
|
-
def base_url(self) -> str:
|
|
88
|
-
return self._credentials.server_url
|
|
89
|
-
|
|
90
|
-
def _fetch_users(self) -> SerializedAsset:
|
|
91
|
-
"""Fetches list of User"""
|
|
92
|
-
return [
|
|
93
|
-
extract_asset(user, TableauAsset.USER)
|
|
94
|
-
for user in TSC.Pager(self._server.users)
|
|
95
|
-
]
|
|
96
|
-
|
|
97
|
-
def _fetch_workbooks(self) -> SerializedAsset:
|
|
98
|
-
"""Fetches list of Workbooks"""
|
|
99
|
-
|
|
100
|
-
return [
|
|
101
|
-
extract_asset(workbook, TableauAsset.WORKBOOK)
|
|
102
|
-
for workbook in TSC.Pager(self._server.workbooks)
|
|
103
|
-
]
|
|
104
|
-
|
|
105
|
-
def _fetch_usages(self, safe_mode: bool) -> SerializedAsset:
|
|
106
|
-
"""Fetches list of Usages"""
|
|
107
|
-
if not safe_mode:
|
|
108
|
-
usages = [
|
|
109
|
-
extract_asset(usage, TableauAsset.USAGE)
|
|
110
|
-
for usage in TSC.Pager(self._server.views, usage=True)
|
|
111
|
-
]
|
|
112
|
-
|
|
113
|
-
return compute_usage_views(usages)
|
|
114
|
-
|
|
115
|
-
return safe_mode_fetch_usage(self)
|
|
116
|
-
|
|
117
|
-
def _fetch_projects(self) -> SerializedAsset:
|
|
118
|
-
"""Fetches list of Projects"""
|
|
119
|
-
return compute_project_path(
|
|
120
|
-
[
|
|
121
|
-
extract_asset(project, TableauAsset.PROJECT)
|
|
122
|
-
for project in TSC.Pager(self._server.projects)
|
|
123
|
-
],
|
|
124
|
-
)
|
|
125
|
-
|
|
126
|
-
def _fetch_workbooks_to_datasource(self) -> SerializedAsset:
|
|
127
|
-
"""Fetches workbooks to datasource"""
|
|
128
|
-
|
|
129
|
-
return self._fetch_paginated_objects(
|
|
130
|
-
TableauAsset.WORKBOOK_TO_DATASOURCE,
|
|
131
|
-
)
|
|
132
|
-
|
|
133
|
-
def _fetch_published_datasources(self) -> SerializedAsset:
|
|
134
|
-
"""Fetches list of published datasources"""
|
|
135
|
-
|
|
136
|
-
return [
|
|
137
|
-
extract_asset(datasource, TableauAsset.PUBLISHED_DATASOURCE)
|
|
138
|
-
for datasource in TSC.Pager(self._server.datasources)
|
|
139
|
-
]
|
|
140
|
-
|
|
141
|
-
def _fetch_datasources(self) -> SerializedAsset:
|
|
142
|
-
"""Fetches both embedded and published datasource"""
|
|
143
|
-
|
|
144
|
-
return self._fetch_paginated_objects(
|
|
145
|
-
TableauAsset.DATASOURCE,
|
|
146
|
-
)
|
|
147
|
-
|
|
148
|
-
def _fetch_fields(self) -> SerializedAsset:
|
|
149
|
-
"""Fetches fields"""
|
|
150
|
-
return self._fetch_paginated_objects(
|
|
151
|
-
TableauAsset.FIELD,
|
|
152
|
-
)
|
|
153
|
-
|
|
154
|
-
def _fetch_custom_sql_queries(self) -> SerializedAsset:
|
|
155
|
-
"""Fetches custom sql queries"""
|
|
156
|
-
|
|
157
|
-
return self._fetch_paginated_objects(
|
|
158
|
-
TableauAsset.CUSTOM_SQL_QUERY,
|
|
159
|
-
)
|
|
160
|
-
|
|
161
|
-
def _fetch_custom_sql_tables(self) -> SerializedAsset:
|
|
162
|
-
"""Fetches custom sql tables"""
|
|
163
|
-
|
|
164
|
-
return self._fetch_paginated_objects(
|
|
165
|
-
TableauAsset.CUSTOM_SQL_TABLE,
|
|
166
|
-
)
|
|
167
|
-
|
|
168
|
-
def _fetch_dashboards(self) -> SerializedAsset:
|
|
169
|
-
"""Fetches dashboards"""
|
|
170
|
-
|
|
171
|
-
return self._fetch_paginated_objects(
|
|
172
|
-
TableauAsset.DASHBOARD,
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
def _fetch_sheets(self) -> SerializedAsset:
|
|
176
|
-
"""Fetches sheets"""
|
|
177
|
-
|
|
178
|
-
return self._fetch_paginated_objects(
|
|
179
|
-
TableauAsset.SHEET,
|
|
180
|
-
)
|
|
181
|
-
|
|
182
|
-
def _fetch_paginated_objects(self, asset: TableauAsset) -> SerializedAsset:
|
|
183
|
-
"""Fetches paginated objects"""
|
|
184
|
-
|
|
185
|
-
return get_paginated_objects(self._server, asset, self._page_size)
|
|
186
|
-
|
|
187
|
-
def fetch(self, asset: TableauAsset) -> SerializedAsset:
|
|
188
|
-
"""Fetches the given asset"""
|
|
189
|
-
logger.info(f"Fetching {asset.name}")
|
|
190
|
-
|
|
191
|
-
if asset == TableauAsset.CUSTOM_SQL_QUERY:
|
|
192
|
-
assets = self._fetch_custom_sql_queries()
|
|
193
|
-
|
|
194
|
-
if asset == TableauAsset.CUSTOM_SQL_TABLE:
|
|
195
|
-
assets = self._fetch_custom_sql_tables()
|
|
196
|
-
|
|
197
|
-
if asset == TableauAsset.DASHBOARD:
|
|
198
|
-
assets = self._fetch_dashboards()
|
|
199
|
-
|
|
200
|
-
if asset == TableauAsset.DATASOURCE:
|
|
201
|
-
assets = self._fetch_datasources()
|
|
202
|
-
|
|
203
|
-
if asset == TableauAsset.FIELD:
|
|
204
|
-
assets = self._fetch_fields()
|
|
205
|
-
|
|
206
|
-
if asset == TableauAsset.PROJECT:
|
|
207
|
-
assets = self._fetch_projects()
|
|
208
|
-
|
|
209
|
-
if asset == TableauAsset.PUBLISHED_DATASOURCE:
|
|
210
|
-
assets = self._fetch_published_datasources()
|
|
211
|
-
|
|
212
|
-
if asset == TableauAsset.SHEET:
|
|
213
|
-
assets = self._fetch_sheets()
|
|
214
|
-
|
|
215
|
-
if asset == TableauAsset.USAGE:
|
|
216
|
-
assets = self._fetch_usages(self._safe_mode)
|
|
217
|
-
|
|
218
|
-
if asset == TableauAsset.USER:
|
|
219
|
-
assets = self._fetch_users()
|
|
220
|
-
|
|
221
|
-
if asset == TableauAsset.WORKBOOK:
|
|
222
|
-
assets = self._fetch_workbooks()
|
|
223
|
-
|
|
224
|
-
if asset == TableauAsset.WORKBOOK_TO_DATASOURCE:
|
|
225
|
-
assets = self._fetch_workbooks_to_datasource()
|
|
226
|
-
|
|
227
|
-
logger.info(f"Fetched {asset.name} ({len(assets)} results)")
|
|
228
|
-
|
|
229
|
-
return assets
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
from collections.abc import Iterator
|
|
2
|
-
from typing import Optional
|
|
3
|
-
|
|
4
|
-
from ....utils import SerializedAsset
|
|
5
|
-
from ..assets import TableauAsset
|
|
6
|
-
from ..gql_fields import QUERY_FIELDS
|
|
7
|
-
from ..tsc_fields import TSC_FIELDS
|
|
8
|
-
|
|
9
|
-
QUERY_TEMPLATE = """
|
|
10
|
-
{{
|
|
11
|
-
{object_type}Connection(first: {page_size}, after: AFTER_TOKEN_SIGNAL) {{
|
|
12
|
-
nodes {{ {query_fields}
|
|
13
|
-
}}
|
|
14
|
-
pageInfo {{
|
|
15
|
-
hasNextPage
|
|
16
|
-
endCursor
|
|
17
|
-
}}
|
|
18
|
-
totalCount
|
|
19
|
-
}}
|
|
20
|
-
}}
|
|
21
|
-
"""
|
|
22
|
-
|
|
23
|
-
RESOURCE_TEMPLATE = "{resource}Connection"
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def get_paginated_objects(
|
|
27
|
-
server,
|
|
28
|
-
asset: TableauAsset,
|
|
29
|
-
page_size: int,
|
|
30
|
-
) -> SerializedAsset:
|
|
31
|
-
assets: SerializedAsset = []
|
|
32
|
-
for query in QUERY_FIELDS[asset]:
|
|
33
|
-
fields = query["fields"].value
|
|
34
|
-
object_type = query["object_type"].value
|
|
35
|
-
query_formatted = QUERY_TEMPLATE.format(
|
|
36
|
-
object_type=object_type,
|
|
37
|
-
page_size=page_size,
|
|
38
|
-
query_fields=fields,
|
|
39
|
-
)
|
|
40
|
-
resource = RESOURCE_TEMPLATE.format(resource=object_type)
|
|
41
|
-
result_pages = query_scroll(server, query_formatted, resource)
|
|
42
|
-
queried_assets = [asset for page in result_pages for asset in page]
|
|
43
|
-
assets.extend(queried_assets)
|
|
44
|
-
return assets
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def query_scroll(
|
|
48
|
-
server,
|
|
49
|
-
query: str,
|
|
50
|
-
resource: str,
|
|
51
|
-
) -> Iterator[SerializedAsset]:
|
|
52
|
-
"""build a tableau query iterator handling pagination and cursor"""
|
|
53
|
-
|
|
54
|
-
def _call(cursor: Optional[str]) -> dict:
|
|
55
|
-
# If cursor is defined it must be quoted else use null token
|
|
56
|
-
token = "null" if cursor is None else f'"{cursor}"'
|
|
57
|
-
query_ = query.replace("AFTER_TOKEN_SIGNAL", token)
|
|
58
|
-
|
|
59
|
-
return server.metadata.query(query_)["data"][resource]
|
|
60
|
-
|
|
61
|
-
cursor = None
|
|
62
|
-
while True:
|
|
63
|
-
payload = _call(cursor)
|
|
64
|
-
yield payload["nodes"]
|
|
65
|
-
|
|
66
|
-
page_info = payload["pageInfo"]
|
|
67
|
-
if page_info["hasNextPage"]:
|
|
68
|
-
cursor = page_info["endCursor"]
|
|
69
|
-
else:
|
|
70
|
-
break
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def extract_asset(asset: dict, asset_type: TableauAsset) -> dict:
|
|
74
|
-
"""Agnostic function extracting dedicated attributes with define asset"""
|
|
75
|
-
return {key: getattr(asset, key) for key in TSC_FIELDS[asset_type]}
|
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
|
-
from typing import Optional
|
|
3
|
-
|
|
4
|
-
from ....utils import from_env
|
|
5
|
-
|
|
6
|
-
AUTH_ERROR_MSG = "Need either user and password or token_name and token"
|
|
7
|
-
|
|
8
|
-
# https://tableau.github.io/server-client-python/docs/api-ref#authentication
|
|
9
|
-
DEFAULT_SERVER_SITE_ID = ""
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
class CredentialsKey(Enum):
|
|
13
|
-
"""Value enum object for the credentials"""
|
|
14
|
-
|
|
15
|
-
TABLEAU_USER = "user"
|
|
16
|
-
TABLEAU_PASSWORD = "password" # noqa: S105
|
|
17
|
-
TABLEAU_TOKEN_NAME = "token_name" # noqa: S105
|
|
18
|
-
TABLEAU_TOKEN = "token" # noqa: S105
|
|
19
|
-
TABLEAU_SITE_ID = "site_id"
|
|
20
|
-
TABLEAU_SERVER_URL = "server_url"
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
CREDENTIALS_ENV: dict[CredentialsKey, str] = {
|
|
24
|
-
CredentialsKey.TABLEAU_USER: "CASTOR_TABLEAU_USER",
|
|
25
|
-
CredentialsKey.TABLEAU_PASSWORD: "CASTOR_TABLEAU_PASSWORD",
|
|
26
|
-
CredentialsKey.TABLEAU_TOKEN_NAME: "CASTOR_TABLEAU_TOKEN_NAME",
|
|
27
|
-
CredentialsKey.TABLEAU_TOKEN: "CASTOR_TABLEAU_TOKEN",
|
|
28
|
-
CredentialsKey.TABLEAU_SITE_ID: "CASTOR_TABLEAU_SITE_ID",
|
|
29
|
-
CredentialsKey.TABLEAU_SERVER_URL: "CASTOR_TABLEAU_SERVER_URL",
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def get_value(
|
|
34
|
-
key: CredentialsKey,
|
|
35
|
-
kwargs: dict,
|
|
36
|
-
optional: bool = False,
|
|
37
|
-
) -> Optional[str]:
|
|
38
|
-
"""
|
|
39
|
-
Returns the value of the given key:
|
|
40
|
-
- from kwargs in priority
|
|
41
|
-
- from ENV if not provided (raises an error if not found in ENV)
|
|
42
|
-
"""
|
|
43
|
-
env_key = CREDENTIALS_ENV[key]
|
|
44
|
-
|
|
45
|
-
return kwargs.get(key.value) or from_env(env_key, optional)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
class CredentialsApi:
|
|
49
|
-
"""ValueObject for the credentials"""
|
|
50
|
-
|
|
51
|
-
def __init__(
|
|
52
|
-
self,
|
|
53
|
-
*,
|
|
54
|
-
server_url: str,
|
|
55
|
-
site_id: Optional[str],
|
|
56
|
-
user: Optional[str],
|
|
57
|
-
password: Optional[str],
|
|
58
|
-
token_name: Optional[str],
|
|
59
|
-
token: Optional[str],
|
|
60
|
-
):
|
|
61
|
-
credentials = self._get_credentials(user, password, token_name, token)
|
|
62
|
-
|
|
63
|
-
self.user = credentials.get(CredentialsKey.TABLEAU_USER)
|
|
64
|
-
self.site_id = site_id if site_id else DEFAULT_SERVER_SITE_ID
|
|
65
|
-
self.server_url = server_url
|
|
66
|
-
self.password = credentials.get(CredentialsKey.TABLEAU_PASSWORD)
|
|
67
|
-
self.token_name = credentials.get(CredentialsKey.TABLEAU_TOKEN_NAME)
|
|
68
|
-
self.token = credentials.get(CredentialsKey.TABLEAU_TOKEN)
|
|
69
|
-
|
|
70
|
-
@staticmethod
|
|
71
|
-
def _get_credentials(
|
|
72
|
-
user: Optional[str],
|
|
73
|
-
password: Optional[str],
|
|
74
|
-
token_name: Optional[str],
|
|
75
|
-
token: Optional[str],
|
|
76
|
-
) -> dict:
|
|
77
|
-
"""Helpers to retrieve credentials,
|
|
78
|
-
if both are given choose user and password authentication method"""
|
|
79
|
-
assert (user and password) or (token_name and token), AUTH_ERROR_MSG
|
|
80
|
-
|
|
81
|
-
if user and password:
|
|
82
|
-
return {
|
|
83
|
-
CredentialsKey.TABLEAU_USER: user,
|
|
84
|
-
CredentialsKey.TABLEAU_PASSWORD: password,
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
return {
|
|
88
|
-
CredentialsKey.TABLEAU_TOKEN_NAME: token_name,
|
|
89
|
-
CredentialsKey.TABLEAU_TOKEN: token,
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
def to_dict(self, hide: bool = False) -> dict[str, str]:
|
|
93
|
-
safe = (
|
|
94
|
-
CredentialsKey.TABLEAU_USER,
|
|
95
|
-
CredentialsKey.TABLEAU_SITE_ID,
|
|
96
|
-
CredentialsKey.TABLEAU_SERVER_URL,
|
|
97
|
-
CredentialsKey.TABLEAU_TOKEN_NAME,
|
|
98
|
-
)
|
|
99
|
-
unsafe = (CredentialsKey.TABLEAU_PASSWORD, CredentialsKey.TABLEAU_TOKEN)
|
|
100
|
-
|
|
101
|
-
def val(k: CredentialsKey, v: str) -> str:
|
|
102
|
-
return "*" + v[-4:] if hide and k in unsafe else v
|
|
103
|
-
|
|
104
|
-
return {a.value: val(a, getattr(self, a.value)) for a in safe + unsafe}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
|
-
from ....utils import SerializedAsset
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def _folder_path(
|
|
7
|
-
projects: SerializedAsset,
|
|
8
|
-
project: dict,
|
|
9
|
-
root: Optional[str] = "",
|
|
10
|
-
) -> str:
|
|
11
|
-
"""Recursive function to compute folder path with list of projects"""
|
|
12
|
-
path = "/" + str(project["name"]) + (root or "")
|
|
13
|
-
if project["parent_id"] is None:
|
|
14
|
-
return path
|
|
15
|
-
|
|
16
|
-
parent_project = next(
|
|
17
|
-
parent_project
|
|
18
|
-
for parent_project in projects
|
|
19
|
-
if parent_project["id"] == project["parent_id"]
|
|
20
|
-
)
|
|
21
|
-
return _folder_path(projects, parent_project, path)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def compute_project_path(projects: SerializedAsset) -> SerializedAsset:
|
|
25
|
-
"""Compute folder path with parent project name"""
|
|
26
|
-
for project in projects:
|
|
27
|
-
project["folder_path"] = _folder_path(projects, project)
|
|
28
|
-
return projects
|