castor-extractor 0.19.0__py3-none-any.whl → 0.19.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of castor-extractor might be problematic. Click here for more details.
- CHANGELOG.md +29 -2
- castor_extractor/file_checker/templates/generic_warehouse.py +1 -1
- castor_extractor/knowledge/notion/client/client.py +44 -80
- castor_extractor/knowledge/notion/client/client_test.py +9 -4
- castor_extractor/knowledge/notion/client/constants.py +1 -0
- castor_extractor/knowledge/notion/client/endpoints.py +1 -1
- castor_extractor/knowledge/notion/client/pagination.py +9 -5
- castor_extractor/quality/soda/assets.py +1 -1
- castor_extractor/quality/soda/client/client.py +30 -83
- castor_extractor/quality/soda/client/credentials.py +0 -11
- castor_extractor/quality/soda/client/endpoints.py +3 -6
- castor_extractor/quality/soda/client/pagination.py +25 -0
- castor_extractor/utils/__init__.py +13 -2
- castor_extractor/utils/client/__init__.py +14 -0
- castor_extractor/utils/client/api/__init__.py +5 -0
- castor_extractor/utils/client/api/auth.py +76 -0
- castor_extractor/utils/client/api/auth_test.py +49 -0
- castor_extractor/utils/client/api/client.py +153 -0
- castor_extractor/utils/client/api/client_test.py +47 -0
- castor_extractor/utils/client/api/pagination.py +83 -0
- castor_extractor/utils/client/api/pagination_test.py +51 -0
- castor_extractor/utils/{safe_request_test.py → client/api/safe_request_test.py} +4 -1
- castor_extractor/utils/client/api/utils.py +9 -0
- castor_extractor/utils/client/api/utils_test.py +16 -0
- castor_extractor/utils/collection.py +34 -2
- castor_extractor/utils/collection_test.py +17 -3
- castor_extractor/utils/pager/__init__.py +0 -1
- castor_extractor/utils/retry.py +44 -0
- castor_extractor/utils/retry_test.py +26 -1
- castor_extractor/utils/salesforce/client.py +44 -49
- castor_extractor/utils/salesforce/client_test.py +2 -2
- castor_extractor/utils/salesforce/pagination.py +33 -0
- castor_extractor/visualization/domo/client/client.py +10 -5
- castor_extractor/visualization/domo/client/credentials.py +1 -1
- castor_extractor/visualization/domo/client/endpoints.py +19 -7
- castor_extractor/visualization/looker/api/credentials.py +1 -1
- castor_extractor/visualization/metabase/client/api/client.py +26 -11
- castor_extractor/visualization/metabase/client/api/credentials.py +1 -1
- castor_extractor/visualization/metabase/client/db/credentials.py +1 -1
- castor_extractor/visualization/mode/client/credentials.py +1 -1
- castor_extractor/visualization/qlik/client/engine/credentials.py +1 -1
- castor_extractor/visualization/salesforce_reporting/client/rest.py +4 -3
- castor_extractor/visualization/sigma/client/client.py +106 -111
- castor_extractor/visualization/sigma/client/credentials.py +11 -1
- castor_extractor/visualization/sigma/client/endpoints.py +1 -1
- castor_extractor/visualization/sigma/client/pagination.py +22 -18
- castor_extractor/visualization/tableau/tests/unit/rest_api/auth_test.py +0 -1
- castor_extractor/visualization/tableau/tests/unit/rest_api/credentials_test.py +0 -3
- castor_extractor/visualization/tableau_revamp/assets.py +11 -0
- castor_extractor/visualization/tableau_revamp/client/client.py +71 -151
- castor_extractor/visualization/tableau_revamp/client/client_metadata_api.py +95 -0
- castor_extractor/visualization/tableau_revamp/client/client_rest_api.py +128 -0
- castor_extractor/visualization/tableau_revamp/client/client_tsc.py +66 -0
- castor_extractor/visualization/tableau_revamp/client/{tsc_fields.py → rest_fields.py} +15 -2
- castor_extractor/visualization/tableau_revamp/constants.py +0 -2
- castor_extractor/visualization/tableau_revamp/extract.py +5 -11
- castor_extractor/warehouse/databricks/api_client.py +239 -0
- castor_extractor/warehouse/databricks/api_client_test.py +15 -0
- castor_extractor/warehouse/databricks/client.py +37 -490
- castor_extractor/warehouse/databricks/client_test.py +1 -99
- castor_extractor/warehouse/databricks/endpoints.py +28 -0
- castor_extractor/warehouse/databricks/lineage.py +141 -0
- castor_extractor/warehouse/databricks/lineage_test.py +34 -0
- castor_extractor/warehouse/databricks/pagination.py +22 -0
- castor_extractor/warehouse/databricks/sql_client.py +90 -0
- castor_extractor/warehouse/databricks/utils.py +44 -1
- castor_extractor/warehouse/databricks/utils_test.py +58 -1
- castor_extractor/warehouse/mysql/client.py +0 -2
- castor_extractor/warehouse/salesforce/client.py +12 -59
- castor_extractor/warehouse/salesforce/pagination.py +34 -0
- castor_extractor/warehouse/sqlserver/client.py +0 -1
- castor_extractor-0.19.6.dist-info/METADATA +903 -0
- {castor_extractor-0.19.0.dist-info → castor_extractor-0.19.6.dist-info}/RECORD +77 -60
- castor_extractor/utils/client/api.py +0 -87
- castor_extractor/utils/client/api_test.py +0 -24
- castor_extractor/utils/pager/pager_on_token.py +0 -52
- castor_extractor/utils/pager/pager_on_token_test.py +0 -73
- castor_extractor/visualization/sigma/client/client_test.py +0 -54
- castor_extractor-0.19.0.dist-info/METADATA +0 -207
- /castor_extractor/utils/{safe_request.py → client/api/safe_request.py} +0 -0
- {castor_extractor-0.19.0.dist-info → castor_extractor-0.19.6.dist-info}/LICENCE +0 -0
- {castor_extractor-0.19.0.dist-info → castor_extractor-0.19.6.dist-info}/WHEEL +0 -0
- {castor_extractor-0.19.0.dist-info → castor_extractor-0.19.6.dist-info}/entry_points.txt +0 -0
|
@@ -1,207 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: castor-extractor
|
|
3
|
-
Version: 0.19.0
|
|
4
|
-
Summary: Extract your metadata assets.
|
|
5
|
-
Home-page: https://www.castordoc.com/
|
|
6
|
-
License: EULA
|
|
7
|
-
Author: Castor
|
|
8
|
-
Author-email: support@castordoc.com
|
|
9
|
-
Requires-Python: >=3.8,<3.13
|
|
10
|
-
Classifier: License :: Other/Proprietary License
|
|
11
|
-
Classifier: Operating System :: OS Independent
|
|
12
|
-
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
Provides-Extra: all
|
|
19
|
-
Provides-Extra: bigquery
|
|
20
|
-
Provides-Extra: databricks
|
|
21
|
-
Provides-Extra: dbt
|
|
22
|
-
Provides-Extra: looker
|
|
23
|
-
Provides-Extra: metabase
|
|
24
|
-
Provides-Extra: mysql
|
|
25
|
-
Provides-Extra: postgres
|
|
26
|
-
Provides-Extra: powerbi
|
|
27
|
-
Provides-Extra: qlik
|
|
28
|
-
Provides-Extra: redshift
|
|
29
|
-
Provides-Extra: snowflake
|
|
30
|
-
Provides-Extra: sqlserver
|
|
31
|
-
Provides-Extra: tableau
|
|
32
|
-
Requires-Dist: cryptography (>=42.0.0,<43.0.0) ; extra == "snowflake"
|
|
33
|
-
Requires-Dist: databricks-sql-connector (>=3.2.0,<4.0.0) ; extra == "databricks" or extra == "all"
|
|
34
|
-
Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
|
|
35
|
-
Requires-Dist: google-auth (>=2,<3)
|
|
36
|
-
Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
|
|
37
|
-
Requires-Dist: google-cloud-storage (>=2,<3)
|
|
38
|
-
Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
|
|
39
|
-
Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
|
|
40
|
-
Requires-Dist: looker-sdk (>=24.0.0,<25.0.0) ; extra == "looker" or extra == "all"
|
|
41
|
-
Requires-Dist: msal (>=1.20.0,<2.0.0) ; extra == "powerbi" or extra == "all"
|
|
42
|
-
Requires-Dist: numpy (<1.25) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "bigquery" or extra == "databricks" or extra == "all")
|
|
43
|
-
Requires-Dist: numpy (<2) ; extra == "bigquery" or extra == "databricks" or extra == "all"
|
|
44
|
-
Requires-Dist: numpy (>=1.26) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "bigquery" or extra == "databricks" or extra == "all")
|
|
45
|
-
Requires-Dist: pandas (<2.1) ; (python_version >= "3.8" and python_version < "3.9") and (extra == "databricks" or extra == "all")
|
|
46
|
-
Requires-Dist: pandas (>=2.1) ; (python_version >= "3.12" and python_version < "3.13") and (extra == "databricks" or extra == "all")
|
|
47
|
-
Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0) ; extra == "metabase" or extra == "postgres" or extra == "redshift" or extra == "all"
|
|
48
|
-
Requires-Dist: pycryptodome (>=3.0.0,<4.0.0) ; extra == "metabase" or extra == "all"
|
|
49
|
-
Requires-Dist: pydantic (>=2.6,<3.0)
|
|
50
|
-
Requires-Dist: pydantic-settings (>=2.2,<3.0)
|
|
51
|
-
Requires-Dist: pymssql (>=2.2.11,<3.0.0) ; extra == "sqlserver" or extra == "all"
|
|
52
|
-
Requires-Dist: pymysql[rsa] (>=1.1.0,<2.0.0) ; extra == "mysql" or extra == "all"
|
|
53
|
-
Requires-Dist: python-dateutil (>=2.0.0,<=3.0.0)
|
|
54
|
-
Requires-Dist: requests (>=2.0.0,<3.0.0)
|
|
55
|
-
Requires-Dist: setuptools (>=72,<73)
|
|
56
|
-
Requires-Dist: snowflake-connector-python (>=3.4.0,<4.0.0) ; extra == "snowflake" or extra == "all"
|
|
57
|
-
Requires-Dist: snowflake-sqlalchemy (!=1.2.5,<2.0.0) ; extra == "snowflake" or extra == "all"
|
|
58
|
-
Requires-Dist: sqlalchemy (>=1.4,<1.5)
|
|
59
|
-
Requires-Dist: sqlalchemy-bigquery[bqstorage] (>=1.0.0,<=2.0.0) ; extra == "bigquery" or extra == "all"
|
|
60
|
-
Requires-Dist: sqlalchemy-redshift (>=0.8.14,<0.9.0) ; extra == "redshift" or extra == "all"
|
|
61
|
-
Requires-Dist: tableauserverclient (==0.25.0) ; extra == "tableau" or extra == "all"
|
|
62
|
-
Requires-Dist: tqdm (>=4.0.0,<5.0.0)
|
|
63
|
-
Requires-Dist: typing-extensions (>=4,<5)
|
|
64
|
-
Requires-Dist: websocket-client (>=1,<2) ; extra == "qlik" or extra == "all"
|
|
65
|
-
Project-URL: Documentation, https://docs.castordoc.com/castor-package/castor-extractor
|
|
66
|
-
Description-Content-Type: text/markdown
|
|
67
|
-
|
|
68
|
-
# Castor Extractor <img src="https://app.castordoc.com/images/castor_icon_dark.svg" width=30 />
|
|
69
|
-
|
|
70
|
-
This library contains utilities to extract your metadata assets into `JSON` or `CSV` files, on your local machine.
|
|
71
|
-
After extraction, those files can be pushed to Castor for ingestion.
|
|
72
|
-
|
|
73
|
-
- Visualization assets are typically:
|
|
74
|
-
- `dashboards`
|
|
75
|
-
- `users`
|
|
76
|
-
- `folders`
|
|
77
|
-
- ...
|
|
78
|
-
|
|
79
|
-
- Warehouse assets are typically:
|
|
80
|
-
- `databases`
|
|
81
|
-
- `schemas`
|
|
82
|
-
- `tables`
|
|
83
|
-
- `columns`
|
|
84
|
-
- `queries`
|
|
85
|
-
- ...
|
|
86
|
-
|
|
87
|
-
It also embeds utilities to help you push your metadata to Castor:
|
|
88
|
-
|
|
89
|
-
- `File Checker` to validate your [generic](https://docs.castordoc.com/integrations/data-warehouses/generic-warehouse) CSV files before pushing to Castor
|
|
90
|
-
- `Uploader` to push extracted files to our Google-Cloud-Storage (GCS)
|
|
91
|
-
|
|
92
|
-
## Table of contents
|
|
93
|
-
|
|
94
|
-
- [Castor Extractor](#castor-extractor-)
|
|
95
|
-
- [Table of contents](#table-of-contents)
|
|
96
|
-
- [Installation](#installation)
|
|
97
|
-
- [Create castor-env](#create-castor-env)
|
|
98
|
-
- [PIP install](#pip-install)
|
|
99
|
-
- [Create the output directory](#create-the-output-directory)
|
|
100
|
-
- [Contact](#contact)
|
|
101
|
-
|
|
102
|
-
## Installation
|
|
103
|
-
|
|
104
|
-
Requirements: **python3.8+**
|
|
105
|
-
<img src="https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" width=20 />
|
|
106
|
-
|
|
107
|
-
### Create castor-env
|
|
108
|
-
|
|
109
|
-
We advise to create a dedicated [Python environment](https://docs.python.org/3/library/venv.html).
|
|
110
|
-
|
|
111
|
-
Here's an example using `Pyenv` and Python `3.8.12`:
|
|
112
|
-
|
|
113
|
-
- Install Pyenv
|
|
114
|
-
|
|
115
|
-
```bash
|
|
116
|
-
brew install pyenv
|
|
117
|
-
brew install pyenv-virtualenv
|
|
118
|
-
```
|
|
119
|
-
|
|
120
|
-
- [optional] Update your `.bashrc` if you encounter this [issue](https://stackoverflow.com/questions/45577194/failed-to-activate-virtualenv-with-pyenv/45578839)
|
|
121
|
-
|
|
122
|
-
```bash
|
|
123
|
-
eval "$(pyenv init -)"
|
|
124
|
-
eval "$(pyenv init --path)"
|
|
125
|
-
eval "$(pyenv virtualenv-init -)"
|
|
126
|
-
```
|
|
127
|
-
|
|
128
|
-
- [optional] Install python 3.8+
|
|
129
|
-
|
|
130
|
-
```bash
|
|
131
|
-
pyenv versions # check your local python installations
|
|
132
|
-
|
|
133
|
-
pyenv install -v 3.8.12 # if none of the installed versions satisfy requirements 8+
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
- Create your virtual env
|
|
137
|
-
|
|
138
|
-
```bash
|
|
139
|
-
pyenv virtualenv 3.8.12 castor-env # create a dedicated env
|
|
140
|
-
pyenv shell castor-env # activate the environment
|
|
141
|
-
|
|
142
|
-
# optional checks
|
|
143
|
-
python --version # should be `3.8.12`
|
|
144
|
-
pyenv version # should be `castor-env`
|
|
145
|
-
```
|
|
146
|
-
|
|
147
|
-
### PIP install
|
|
148
|
-
|
|
149
|
-
⚠️ `castor-env` must be created AND activated first.
|
|
150
|
-
|
|
151
|
-
```bash
|
|
152
|
-
pyenv shell castor-env
|
|
153
|
-
(castor-env) $ # this means the environment is now active
|
|
154
|
-
```
|
|
155
|
-
|
|
156
|
-
ℹ️ please upgrade `PIP` before installing Castor.
|
|
157
|
-
|
|
158
|
-
```bash
|
|
159
|
-
pip install --upgrade pip
|
|
160
|
-
```
|
|
161
|
-
|
|
162
|
-
Run the following command to install `castor-extractor`:
|
|
163
|
-
|
|
164
|
-
```bash
|
|
165
|
-
pip install castor-extractor
|
|
166
|
-
```
|
|
167
|
-
|
|
168
|
-
Depending on your use case, you can also install one of the following `extras`:
|
|
169
|
-
|
|
170
|
-
```bash
|
|
171
|
-
pip install castor-extractor[bigquery]
|
|
172
|
-
pip install castor-extractor[looker]
|
|
173
|
-
pip install castor-extractor[metabase]
|
|
174
|
-
pip install castor-extractor[mysql]
|
|
175
|
-
pip install castor-extractor[powerbi]
|
|
176
|
-
pip install castor-extractor[qlik]
|
|
177
|
-
pip install castor-extractor[postgres]
|
|
178
|
-
pip install castor-extractor[redshift]
|
|
179
|
-
pip install castor-extractor[snowflake]
|
|
180
|
-
pip install castor-extractor[sqlserver]
|
|
181
|
-
pip install castor-extractor[tableau]
|
|
182
|
-
```
|
|
183
|
-
|
|
184
|
-
### Create the output directory
|
|
185
|
-
|
|
186
|
-
```bash
|
|
187
|
-
mkdir /tmp/castor
|
|
188
|
-
```
|
|
189
|
-
|
|
190
|
-
You will provide this path in the `extraction` scripts as follows:
|
|
191
|
-
|
|
192
|
-
```bash
|
|
193
|
-
castor-extract-bigquery --output=/tmp/castor
|
|
194
|
-
```
|
|
195
|
-
|
|
196
|
-
Alternatively, you can also set the following `ENV` in your `bashrc`:
|
|
197
|
-
|
|
198
|
-
```bash
|
|
199
|
-
export CASTOR_OUTPUT_DIRECTORY="/tmp/castor"
|
|
200
|
-
````
|
|
201
|
-
|
|
202
|
-
## Contact
|
|
203
|
-
|
|
204
|
-
For any questions or bug report, contact us at [support@castordoc.com](mailto:support@castordoc.com)
|
|
205
|
-
|
|
206
|
-
[Castor](https://castordoc.com) helps you find, understand, use your data assets
|
|
207
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|