pebblous-dataclinic-gw-databricks 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. pebblous_dataclinic_gw_databricks-0.1.1/.gitignore +7 -0
  2. pebblous_dataclinic_gw_databricks-0.1.1/CHANGELOG.md +17 -0
  3. pebblous_dataclinic_gw_databricks-0.1.1/LICENSE +24 -0
  4. pebblous_dataclinic_gw_databricks-0.1.1/PKG-INFO +287 -0
  5. pebblous_dataclinic_gw_databricks-0.1.1/README.md +251 -0
  6. pebblous_dataclinic_gw_databricks-0.1.1/pyproject.toml +77 -0
  7. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/__init__.py +59 -0
  8. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/_version.py +1 -0
  9. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/auth.py +38 -0
  10. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/client.py +461 -0
  11. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/context.py +98 -0
  12. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/datasets.py +49 -0
  13. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/exceptions.py +105 -0
  14. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/logs.py +58 -0
  15. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/models.py +337 -0
  16. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/py.typed +0 -0
  17. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/results.py +54 -0
  18. pebblous_dataclinic_gw_databricks-0.1.1/src/pebblous_dataclinic_gw_databricks/transport.py +185 -0
  19. pebblous_dataclinic_gw_databricks-0.1.1/tests/conftest.py +55 -0
  20. pebblous_dataclinic_gw_databricks-0.1.1/tests/test_auth.py +138 -0
  21. pebblous_dataclinic_gw_databricks-0.1.1/tests/test_client.py +244 -0
  22. pebblous_dataclinic_gw_databricks-0.1.1/tests/test_context_results.py +132 -0
@@ -0,0 +1,7 @@
1
+ .venv/
2
+ dist/
3
+ build/
4
+ *.egg-info/
5
+ __pycache__/
6
+ .pytest_cache/
7
+ .pypirc
@@ -0,0 +1,17 @@
1
+ # Changelog
2
+
3
+ ## 0.1.1 — 2026-11-18
4
+
5
+ Same package as 0.1.0, published by the pebblous.ai PyPI account. PyPI keeps a deleted
6
+ filename forever, so 0.1.0 cannot be uploaded again.
7
+
8
+ ## 0.1.0 — 2026-11-18
9
+
10
+ First release.
11
+
12
+ - `DataClinicGateway` with `from_databricks()` (Sharing Identifier, workspace ID and API key secret detection).
13
+ - DataClinic membership: `verify_member()`, `login()`, `logout()`, `member_session`. `analyze()` attaches `X-DataClinic-Member`.
14
+ - Datasets: `list_datasets()`, `display_datasets()`, `download_dataset()`, `ensure_dataset()`.
15
+ - Analysis: `analyze()`, `analysis_status()`, `analysis_logs()`, `wait()` with incremental logs, `stream_logs()` (SSE).
16
+ - Delta Sharing: `register_sharing()`, `analysis_runs()`, `analysis_results()`.
17
+ - Exceptions per Gateway status, no redirects, network retries that reuse the same `request_id`.
@@ -0,0 +1,24 @@
1
+ Copyright (c) 2026 Pebblous Inc. All rights reserved.
2
+
3
+ pebblous-dataclinic-gw-databricks ("the Software") is proprietary software of Pebblous Inc.
4
+
5
+ Permission is granted to download, install and run the Software, without modification, solely to
6
+ access the Pebblous DataClinic service from your own Databricks workspace or Python environment,
7
+ subject to the Pebblous DataClinic terms of service.
8
+
9
+ You may not, without prior written permission from Pebblous Inc.:
10
+
11
+ 1. copy, modify, merge, translate or create derivative works of the Software, except as needed
12
+ to run it as permitted above;
13
+ 2. distribute, sublicense, sell, rent or otherwise make the Software available to third parties,
14
+ other than through the official Python Package Index distribution;
15
+ 3. use the Software to access any service other than the Pebblous DataClinic service, or to build
16
+ a competing product or service.
17
+
18
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
19
+ NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20
+ NONINFRINGEMENT. IN NO EVENT SHALL PEBBLOUS INC. BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23
+
24
+ Contact: dev@pebblous.ai
@@ -0,0 +1,287 @@
1
+ Metadata-Version: 2.5
2
+ Name: pebblous-dataclinic-gw-databricks
3
+ Version: 0.1.1
4
+ Summary: Run Pebblous DataClinic image-data quality analysis from a Databricks notebook through the DataClinic Gateway.
5
+ Project-URL: Homepage, https://dataclinic.ai
6
+ Project-URL: About DataClinic, https://dataclinic.ai/en/data-clinic/about
7
+ Project-URL: Company, https://www.pebblous.ai
8
+ Author-email: Pebblous <dev@pebblous.ai>
9
+ Maintainer-email: Pebblous <dev@pebblous.ai>
10
+ License-Expression: LicenseRef-Pebblous-Proprietary
11
+ License-File: LICENSE
12
+ Keywords: data-quality,databricks,dataclinic,delta-sharing,embedding,huggingface,image-data,pebblous
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Topic :: Scientific/Engineering :: Image Recognition
25
+ Classifier: Typing :: Typed
26
+ Requires-Python: >=3.10
27
+ Requires-Dist: requests<3,>=2.31
28
+ Provides-Extra: databricks
29
+ Requires-Dist: databricks-sdk>=0.30; extra == 'databricks'
30
+ Provides-Extra: dev
31
+ Requires-Dist: build>=1.2; extra == 'dev'
32
+ Requires-Dist: pytest>=8; extra == 'dev'
33
+ Requires-Dist: responses>=0.25; extra == 'dev'
34
+ Requires-Dist: twine>=6.1; extra == 'dev'
35
+ Description-Content-Type: text/markdown
36
+
37
+ # pebblous-dataclinic-gw-databricks
38
+
39
+ Run [Pebblous DataClinic](https://dataclinic.ai) image-data quality analysis from a Databricks notebook.
40
+
41
+ This package is the client for the **Pebblous DataClinic Gateway**. Your notebook asks the Gateway to
42
+ prepare a dataset and run the DataClinic analyzer on Pebblous GPUs. The results (image embeddings,
43
+ density and k-nearest neighbours) come back to your workspace through Delta Sharing. You never
44
+ handle Pebblous source code or credentials, and your notebook never handles raw HTTP.
45
+
46
+ ```text
47
+ Your Databricks notebook ──► Pebblous Gateway ──► Databricks Jobs (download, GPU analysis)
48
+ ▲ │
49
+ └────────────── Delta Sharing (results) ◄─────────┘
50
+ ```
51
+
52
+ - **Installs one dependency:** `requests`. `databricks-sdk` is optional.
53
+ - **Collects Databricks context for you:** it reads your Metastore Sharing Identifier and workspace ID automatically.
54
+ - **Signs you in once:** you log in with your dataclinic.ai account, and the member token stays in memory only.
55
+ - **Handles waiting and logs:** it polls run status and shows only the new part of each log.
56
+ - **Raises clear exceptions:** each Gateway error maps to an exception class you can catch.
57
+
58
+ > **Status:** alpha (0.1). The API may change before 1.0.
59
+
60
+ ---
61
+
62
+ ## Requirements
63
+
64
+ | | |
65
+ | --- | --- |
66
+ | Python | 3.10 or newer |
67
+ | Runtime | Databricks notebook (Unity Catalog enabled) or any Python 3.10+ |
68
+ | Access | A Gateway URL and API key from Pebblous, and a [dataclinic.ai](https://dataclinic.ai) account (the Free plan works) |
69
+
70
+ ## Install
71
+
72
+ In a Databricks notebook:
73
+
74
+ ```python
75
+ %pip install "pebblous-dataclinic-gw-databricks[databricks]"
76
+ dbutils.library.restartPython()
77
+ ```
78
+
79
+ Anywhere else:
80
+
81
+ ```bash
82
+ pip install pebblous-dataclinic-gw-databricks
83
+ ```
84
+
85
+ ## Quick start (Databricks notebook)
86
+
87
+ ```python
88
+ from pebblous_dataclinic_gw_databricks import DataClinicGateway
89
+
90
+ # 1. Connect. The Sharing Identifier and workspace ID are detected from the notebook.
91
+ # The API key is read from the secret dataclinic/api_key unless you pass api_key=...
92
+ clinic = DataClinicGateway.from_databricks(gateway_url="https://<your-gateway-host>")
93
+ clinic.health()
94
+
95
+ # 2. Sign in with your dataclinic.ai account. The password is asked with a hidden prompt.
96
+ clinic.login()
97
+
98
+ # 3. Choose a dataset and make sure it is downloaded (no GPU used here).
99
+ clinic.display_datasets(refresh=True)
100
+ clinic.ensure_dataset("beans", max_images=300, show_logs=True)
101
+
102
+ # 4. Run the GPU analysis and follow its log.
103
+ run = clinic.analyze(preset="beans", max_images=300, wait=True, show_logs=True)
104
+ print(run.analysis_id, run.status)
105
+
106
+ # 5. Receive the results through Delta Sharing.
107
+ clinic.register_sharing()
108
+ ```
109
+
110
+ Then create the shared catalog once, in SQL:
111
+
112
+ ```sql
113
+ CREATE CATALOG IF NOT EXISTS pebblous_dataclinic_shared
114
+ USING SHARE Pebblous.dataclinic_demo_share;
115
+ ```
116
+
117
+ And read your results:
118
+
119
+ ```python
120
+ display(clinic.analysis_runs())
121
+ display(clinic.analysis_results(run.analysis_id))
122
+ ```
123
+
124
+ ## DataClinic membership
125
+
126
+ The Gateway can require a [dataclinic.ai](https://dataclinic.ai) account before it runs the GPU analysis.
127
+ Every active member qualifies, including the Free plan.
128
+
129
+ ```python
130
+ member = clinic.login() # prompts for email and password
131
+ member = clinic.login("me@company.com") # prompts for the password only
132
+ member = clinic.verify_member(email, password) # non-interactive
133
+
134
+ clinic.member_session # MemberSession(member_id=..., plan=..., expires_at=...) or None
135
+ clinic.logout() # forget the token
136
+ ```
137
+
138
+ | Method | What it does |
139
+ | --- | --- |
140
+ | `verify_member(email, password)` | Calls the Gateway member API (`POST /api/v1/members/verify`) and keeps the returned member token in memory |
141
+ | `login(email=None)` | Notebook login. It uses the secrets `dataclinic/member_email` and `dataclinic/member_password` if both exist. Otherwise it asks for the email and reads the password with `getpass` |
142
+ | `member_session` | The current `MemberSession`, kept in memory only. It is `None` before login, after logout, and after expiry |
143
+ | `analyze()` | Adds the `X-DataClinic-Member` header automatically when you are signed in |
144
+ | `logout()` | Removes the member token from memory |
145
+
146
+ - The password is sent once to the Gateway and never stored. The SDK keeps no copy.
147
+ - The token is left out of `repr()`, printing and exception messages.
148
+ - A token lasts about one hour. After it expires, `analyze()` raises `MemberAuthRequiredError`; call `login()` again.
149
+ - If the Gateway does not use membership checks, `login()` returns `None` and analysis works without it.
150
+ - If the Gateway URL is plain `http://` and not localhost, the SDK warns with `InsecureGatewayWarning` before it sends the password.
151
+
152
+ To skip the prompt in scheduled jobs, store the credentials as Databricks secrets:
153
+
154
+ ```bash
155
+ databricks secrets put-secret dataclinic member_email
156
+ databricks secrets put-secret dataclinic member_password
157
+ ```
158
+
159
+ ## Datasets
160
+
161
+ ```python
162
+ catalog = clinic.list_datasets(refresh=True)
163
+ catalog.volume_datasets # ['demo'] - folders already in the Pebblous Volume
164
+ catalog.presets # Hugging Face presets with cache state
165
+ beans = catalog.get("beans")
166
+ beans.downloaded_images # images ready for analysis now
167
+ beans.ready_for(300) # True if 300 images can be analysed without a download
168
+ catalog.analyzable() # presets you can analyse right away
169
+ ```
170
+
171
+ Hugging Face datasets are **downloaded first, analysed second**. The GPU job never downloads images.
172
+
173
+ ```python
174
+ download = clinic.ensure_dataset("mnist", max_images=500) # waits by default
175
+ download.status # READY or SUCCEEDED
176
+ ```
177
+
178
+ `download_dataset()` returns `status="READY"` and `run_id=None` when the Pebblous Volume already
179
+ holds enough images. That counts as success, so no job is started.
180
+
181
+ ## Analysis
182
+
183
+ ```python
184
+ run = clinic.analyze() # the Pebblous "demo" folder
185
+ run = clinic.analyze(preset="beans", max_images=300) # a downloaded preset
186
+ run.run_id # Databricks run number, used for status and logs
187
+ run.analysis_id # key of your rows in the shared result tables
188
+
189
+ clinic.wait(run.run_id, show_logs=True) # resume watching at any time
190
+ clinic.analysis_status(run.run_id)
191
+ clinic.analysis_logs(run.run_id).text
192
+ ```
193
+
194
+ Live log stream (server-sent events):
195
+
196
+ ```python
197
+ for event in clinic.stream_logs(run.run_id):
198
+ if event.type == "log":
199
+ print(event.text, end="")
200
+ elif event.type == "end":
201
+ print("\nfinished:", event.status)
202
+ ```
203
+
204
+ A `request_id` (UUID) is generated for every request and reused when the SDK retries after a network
205
+ error, so a retry never starts a second job. Pass your own `request_id=` to make a cell safe to re-run.
206
+
207
+ ## Results
208
+
209
+ `analysis_runs()` and `analysis_results()` read the shared views with Spark. They make no Gateway call.
210
+
211
+ | View | Content |
212
+ | --- | --- |
213
+ | `clinic_runs` | One row per analysis: dataset, image count, status, duration |
214
+ | `clinic_analysis` | One row per image: `embedding`, `norm`, `density`, `nearest_id`, `nearest_distance`, `knn` |
215
+
216
+ The views only show rows that belong to your Metastore. Pebblous filters them on the share side
217
+ with `CURRENT_RECIPIENT`.
218
+
219
+ ## Plain Python
220
+
221
+ Outside Databricks, pass the context yourself:
222
+
223
+ ```python
224
+ clinic = DataClinicGateway(
225
+ gateway_url="https://<your-gateway-host>",
226
+ api_key="...",
227
+ sharing_identifier="aws:us-west-2:00000000-0000-0000-0000-000000000000",
228
+ workspace_id=1234567890123456,
229
+ )
230
+ ```
231
+
232
+ Everything except `analysis_runs()` and `analysis_results()` works the same way.
233
+
234
+ ## Errors
235
+
236
+ | Exception | HTTP | Typical cause |
237
+ | --- | --- | --- |
238
+ | `InvalidRequestError` | 400, 422 | Unknown preset, image count over the limit |
239
+ | `DatasetNotDownloadedError` | 400 | `analyze()` before `ensure_dataset()` |
240
+ | `AuthenticationError` | 401 | Wrong API key, or wrong dataclinic.ai email/password |
241
+ | `MemberAuthRequiredError` | 401 | Membership required, or the session expired. Call `login()` |
242
+ | `AuthorizationError` | 403 | Not a dataclinic.ai member, a disabled account, or a token issued for another Metastore |
243
+ | `ResourceNotFoundError` | 404 | Unknown run, or a run started by another Metastore |
244
+ | `RequestConflictError` | 409 | `request_id` reused for a different request |
245
+ | `RateLimitError` | 429 | Another run is active, or too many login attempts. Not retried automatically |
246
+ | `GatewayUnavailableError` | 502, 503 | Gateway cannot reach Databricks, Hugging Face or dataclinic.ai |
247
+ | `GatewayConnectionError` | – | Network problem between you and the Gateway |
248
+ | `RunFailedError` | – | `wait()` saw `FAILED` or `CANCELED`. The final run is in `.run` |
249
+ | `WaitTimeoutError` | – | `wait()` stopped watching. The run continues; call `wait()` again |
250
+
251
+ All of them inherit from `DataClinicError`, which has `status_code` and `detail` attributes.
252
+
253
+ ## Security notes
254
+
255
+ - The package contains no Pebblous credentials. Keep your Gateway API key in a Databricks secret.
256
+ - The SDK does not follow HTTP redirects, so your API key is never sent to another host.
257
+ - The API key, password and member token never appear in `repr()`, logs or exceptions.
258
+ - Use an `https://` Gateway URL.
259
+
260
+ ## API summary
261
+
262
+ | Method | Gateway endpoint |
263
+ | --- | --- |
264
+ | `health()` | `GET /health` |
265
+ | `list_datasets(refresh=False)`, `display_datasets()` | `GET /api/v1/datasets` |
266
+ | `verify_member(email, password)`, `login()` | `POST /api/v1/members/verify` |
267
+ | `logout()`, `member_session` | (local) |
268
+ | `download_dataset(...)`, `ensure_dataset(...)` | `POST /api/v1/downloads` |
269
+ | `download_status(run_id)`, `download_logs(run_id)` | `GET /api/v1/downloads/{run_id}[/logs]` |
270
+ | `analyze(...)` | `POST /api/v1/analyses` |
271
+ | `analysis_status(run_id)`, `analysis_logs(run_id)` | `GET /api/v1/analyses/{run_id}[/logs]` |
272
+ | `wait(run_id, kind)` | status and log polling |
273
+ | `stream_logs(run_id, kind)` | `GET .../logs?follow=true` (SSE) |
274
+ | `register_sharing()` | `POST /api/v1/sharing/register` |
275
+ | `analysis_runs()`, `analysis_results(analysis_id)` | Spark, shared catalog |
276
+
277
+ ## Support
278
+
279
+ - DataClinic: <https://dataclinic.ai>
280
+ - Pebblous: <https://www.pebblous.ai>
281
+ - Email: dev@pebblous.ai
282
+
283
+ ## License
284
+
285
+ Proprietary. Copyright (c) 2026 Pebblous Inc. You may install and run this package, unmodified, to use
286
+ the Pebblous DataClinic service, subject to the DataClinic terms of service. See the `LICENSE` file in
287
+ the distribution for details.
@@ -0,0 +1,251 @@
1
+ # pebblous-dataclinic-gw-databricks
2
+
3
+ Run [Pebblous DataClinic](https://dataclinic.ai) image-data quality analysis from a Databricks notebook.
4
+
5
+ This package is the client for the **Pebblous DataClinic Gateway**. Your notebook asks the Gateway to
6
+ prepare a dataset and run the DataClinic analyzer on Pebblous GPUs. The results (image embeddings,
7
+ density and k-nearest neighbours) come back to your workspace through Delta Sharing. You never
8
+ handle Pebblous source code or credentials, and your notebook never handles raw HTTP.
9
+
10
+ ```text
11
+ Your Databricks notebook ──► Pebblous Gateway ──► Databricks Jobs (download, GPU analysis)
12
+ ▲ │
13
+ └────────────── Delta Sharing (results) ◄─────────┘
14
+ ```
15
+
16
+ - **Installs one dependency:** `requests`. `databricks-sdk` is optional.
17
+ - **Collects Databricks context for you:** it reads your Metastore Sharing Identifier and workspace ID automatically.
18
+ - **Signs you in once:** you log in with your dataclinic.ai account, and the member token stays in memory only.
19
+ - **Handles waiting and logs:** it polls run status and shows only the new part of each log.
20
+ - **Raises clear exceptions:** each Gateway error maps to an exception class you can catch.
21
+
22
+ > **Status:** alpha (0.1). The API may change before 1.0.
23
+
24
+ ---
25
+
26
+ ## Requirements
27
+
28
+ | | |
29
+ | --- | --- |
30
+ | Python | 3.10 or newer |
31
+ | Runtime | Databricks notebook (Unity Catalog enabled) or any Python 3.10+ |
32
+ | Access | A Gateway URL and API key from Pebblous, and a [dataclinic.ai](https://dataclinic.ai) account (the Free plan works) |
33
+
34
+ ## Install
35
+
36
+ In a Databricks notebook:
37
+
38
+ ```python
39
+ %pip install "pebblous-dataclinic-gw-databricks[databricks]"
40
+ dbutils.library.restartPython()
41
+ ```
42
+
43
+ Anywhere else:
44
+
45
+ ```bash
46
+ pip install pebblous-dataclinic-gw-databricks
47
+ ```
48
+
49
+ ## Quick start (Databricks notebook)
50
+
51
+ ```python
52
+ from pebblous_dataclinic_gw_databricks import DataClinicGateway
53
+
54
+ # 1. Connect. The Sharing Identifier and workspace ID are detected from the notebook.
55
+ # The API key is read from the secret dataclinic/api_key unless you pass api_key=...
56
+ clinic = DataClinicGateway.from_databricks(gateway_url="https://<your-gateway-host>")
57
+ clinic.health()
58
+
59
+ # 2. Sign in with your dataclinic.ai account. The password is asked with a hidden prompt.
60
+ clinic.login()
61
+
62
+ # 3. Choose a dataset and make sure it is downloaded (no GPU used here).
63
+ clinic.display_datasets(refresh=True)
64
+ clinic.ensure_dataset("beans", max_images=300, show_logs=True)
65
+
66
+ # 4. Run the GPU analysis and follow its log.
67
+ run = clinic.analyze(preset="beans", max_images=300, wait=True, show_logs=True)
68
+ print(run.analysis_id, run.status)
69
+
70
+ # 5. Receive the results through Delta Sharing.
71
+ clinic.register_sharing()
72
+ ```
73
+
74
+ Then create the shared catalog once, in SQL:
75
+
76
+ ```sql
77
+ CREATE CATALOG IF NOT EXISTS pebblous_dataclinic_shared
78
+ USING SHARE Pebblous.dataclinic_demo_share;
79
+ ```
80
+
81
+ And read your results:
82
+
83
+ ```python
84
+ display(clinic.analysis_runs())
85
+ display(clinic.analysis_results(run.analysis_id))
86
+ ```
87
+
88
+ ## DataClinic membership
89
+
90
+ The Gateway can require a [dataclinic.ai](https://dataclinic.ai) account before it runs the GPU analysis.
91
+ Every active member qualifies, including the Free plan.
92
+
93
+ ```python
94
+ member = clinic.login() # prompts for email and password
95
+ member = clinic.login("me@company.com") # prompts for the password only
96
+ member = clinic.verify_member(email, password) # non-interactive
97
+
98
+ clinic.member_session # MemberSession(member_id=..., plan=..., expires_at=...) or None
99
+ clinic.logout() # forget the token
100
+ ```
101
+
102
+ | Method | What it does |
103
+ | --- | --- |
104
+ | `verify_member(email, password)` | Calls the Gateway member API (`POST /api/v1/members/verify`) and keeps the returned member token in memory |
105
+ | `login(email=None)` | Notebook login. It uses the secrets `dataclinic/member_email` and `dataclinic/member_password` if both exist. Otherwise it asks for the email and reads the password with `getpass` |
106
+ | `member_session` | The current `MemberSession`, kept in memory only. It is `None` before login, after logout, and after expiry |
107
+ | `analyze()` | Adds the `X-DataClinic-Member` header automatically when you are signed in |
108
+ | `logout()` | Removes the member token from memory |
109
+
110
+ - The password is sent once to the Gateway and never stored. The SDK keeps no copy.
111
+ - The token is left out of `repr()`, printing and exception messages.
112
+ - A token lasts about one hour. After it expires, `analyze()` raises `MemberAuthRequiredError`; call `login()` again.
113
+ - If the Gateway does not use membership checks, `login()` returns `None` and analysis works without it.
114
+ - If the Gateway URL is plain `http://` and not localhost, the SDK warns with `InsecureGatewayWarning` before it sends the password.
115
+
116
+ To skip the prompt in scheduled jobs, store the credentials as Databricks secrets:
117
+
118
+ ```bash
119
+ databricks secrets put-secret dataclinic member_email
120
+ databricks secrets put-secret dataclinic member_password
121
+ ```
122
+
123
+ ## Datasets
124
+
125
+ ```python
126
+ catalog = clinic.list_datasets(refresh=True)
127
+ catalog.volume_datasets # ['demo'] - folders already in the Pebblous Volume
128
+ catalog.presets # Hugging Face presets with cache state
129
+ beans = catalog.get("beans")
130
+ beans.downloaded_images # images ready for analysis now
131
+ beans.ready_for(300) # True if 300 images can be analysed without a download
132
+ catalog.analyzable() # presets you can analyse right away
133
+ ```
134
+
135
+ Hugging Face datasets are **downloaded first, analysed second**. The GPU job never downloads images.
136
+
137
+ ```python
138
+ download = clinic.ensure_dataset("mnist", max_images=500) # waits by default
139
+ download.status # READY or SUCCEEDED
140
+ ```
141
+
142
+ `download_dataset()` returns `status="READY"` and `run_id=None` when the Pebblous Volume already
143
+ holds enough images. That counts as success, so no job is started.
144
+
145
+ ## Analysis
146
+
147
+ ```python
148
+ run = clinic.analyze() # the Pebblous "demo" folder
149
+ run = clinic.analyze(preset="beans", max_images=300) # a downloaded preset
150
+ run.run_id # Databricks run number, used for status and logs
151
+ run.analysis_id # key of your rows in the shared result tables
152
+
153
+ clinic.wait(run.run_id, show_logs=True) # resume watching at any time
154
+ clinic.analysis_status(run.run_id)
155
+ clinic.analysis_logs(run.run_id).text
156
+ ```
157
+
158
+ Live log stream (server-sent events):
159
+
160
+ ```python
161
+ for event in clinic.stream_logs(run.run_id):
162
+ if event.type == "log":
163
+ print(event.text, end="")
164
+ elif event.type == "end":
165
+ print("\nfinished:", event.status)
166
+ ```
167
+
168
+ A `request_id` (UUID) is generated for every request and reused when the SDK retries after a network
169
+ error, so a retry never starts a second job. Pass your own `request_id=` to make a cell safe to re-run.
170
+
171
+ ## Results
172
+
173
+ `analysis_runs()` and `analysis_results()` read the shared views with Spark. They make no Gateway call.
174
+
175
+ | View | Content |
176
+ | --- | --- |
177
+ | `clinic_runs` | One row per analysis: dataset, image count, status, duration |
178
+ | `clinic_analysis` | One row per image: `embedding`, `norm`, `density`, `nearest_id`, `nearest_distance`, `knn` |
179
+
180
+ The views only show rows that belong to your Metastore. Pebblous filters them on the share side
181
+ with `CURRENT_RECIPIENT`.
182
+
183
+ ## Plain Python
184
+
185
+ Outside Databricks, pass the context yourself:
186
+
187
+ ```python
188
+ clinic = DataClinicGateway(
189
+ gateway_url="https://<your-gateway-host>",
190
+ api_key="...",
191
+ sharing_identifier="aws:us-west-2:00000000-0000-0000-0000-000000000000",
192
+ workspace_id=1234567890123456,
193
+ )
194
+ ```
195
+
196
+ Everything except `analysis_runs()` and `analysis_results()` works the same way.
197
+
198
+ ## Errors
199
+
200
+ | Exception | HTTP | Typical cause |
201
+ | --- | --- | --- |
202
+ | `InvalidRequestError` | 400, 422 | Unknown preset, image count over the limit |
203
+ | `DatasetNotDownloadedError` | 400 | `analyze()` before `ensure_dataset()` |
204
+ | `AuthenticationError` | 401 | Wrong API key, or wrong dataclinic.ai email/password |
205
+ | `MemberAuthRequiredError` | 401 | Membership required, or the session expired. Call `login()` |
206
+ | `AuthorizationError` | 403 | Not a dataclinic.ai member, a disabled account, or a token issued for another Metastore |
207
+ | `ResourceNotFoundError` | 404 | Unknown run, or a run started by another Metastore |
208
+ | `RequestConflictError` | 409 | `request_id` reused for a different request |
209
+ | `RateLimitError` | 429 | Another run is active, or too many login attempts. Not retried automatically |
210
+ | `GatewayUnavailableError` | 502, 503 | Gateway cannot reach Databricks, Hugging Face or dataclinic.ai |
211
+ | `GatewayConnectionError` | – | Network problem between you and the Gateway |
212
+ | `RunFailedError` | – | `wait()` saw `FAILED` or `CANCELED`. The final run is in `.run` |
213
+ | `WaitTimeoutError` | – | `wait()` stopped watching. The run continues; call `wait()` again |
214
+
215
+ All of them inherit from `DataClinicError`, which has `status_code` and `detail` attributes.
216
+
217
+ ## Security notes
218
+
219
+ - The package contains no Pebblous credentials. Keep your Gateway API key in a Databricks secret.
220
+ - The SDK does not follow HTTP redirects, so your API key is never sent to another host.
221
+ - The API key, password and member token never appear in `repr()`, logs or exceptions.
222
+ - Use an `https://` Gateway URL.
223
+
224
+ ## API summary
225
+
226
+ | Method | Gateway endpoint |
227
+ | --- | --- |
228
+ | `health()` | `GET /health` |
229
+ | `list_datasets(refresh=False)`, `display_datasets()` | `GET /api/v1/datasets` |
230
+ | `verify_member(email, password)`, `login()` | `POST /api/v1/members/verify` |
231
+ | `logout()`, `member_session` | (local) |
232
+ | `download_dataset(...)`, `ensure_dataset(...)` | `POST /api/v1/downloads` |
233
+ | `download_status(run_id)`, `download_logs(run_id)` | `GET /api/v1/downloads/{run_id}[/logs]` |
234
+ | `analyze(...)` | `POST /api/v1/analyses` |
235
+ | `analysis_status(run_id)`, `analysis_logs(run_id)` | `GET /api/v1/analyses/{run_id}[/logs]` |
236
+ | `wait(run_id, kind)` | status and log polling |
237
+ | `stream_logs(run_id, kind)` | `GET .../logs?follow=true` (SSE) |
238
+ | `register_sharing()` | `POST /api/v1/sharing/register` |
239
+ | `analysis_runs()`, `analysis_results(analysis_id)` | Spark, shared catalog |
240
+
241
+ ## Support
242
+
243
+ - DataClinic: <https://dataclinic.ai>
244
+ - Pebblous: <https://www.pebblous.ai>
245
+ - Email: dev@pebblous.ai
246
+
247
+ ## License
248
+
249
+ Proprietary. Copyright (c) 2026 Pebblous Inc. You may install and run this package, unmodified, to use
250
+ the Pebblous DataClinic service, subject to the DataClinic terms of service. See the `LICENSE` file in
251
+ the distribution for details.
@@ -0,0 +1,77 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.27"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "pebblous-dataclinic-gw-databricks"
7
+ dynamic = ["version"]
8
+ description = "Run Pebblous DataClinic image-data quality analysis from a Databricks notebook through the DataClinic Gateway."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "LicenseRef-Pebblous-Proprietary"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Pebblous", email = "dev@pebblous.ai" }]
14
+ maintainers = [{ name = "Pebblous", email = "dev@pebblous.ai" }]
15
+ keywords = [
16
+ "databricks",
17
+ "dataclinic",
18
+ "pebblous",
19
+ "data-quality",
20
+ "image-data",
21
+ "embedding",
22
+ "delta-sharing",
23
+ "huggingface",
24
+ ]
25
+ classifiers = [
26
+ "Development Status :: 3 - Alpha",
27
+ "Intended Audience :: Developers",
28
+ "Intended Audience :: Science/Research",
29
+ "Operating System :: OS Independent",
30
+ "Programming Language :: Python :: 3",
31
+ "Programming Language :: Python :: 3 :: Only",
32
+ "Programming Language :: Python :: 3.10",
33
+ "Programming Language :: Python :: 3.11",
34
+ "Programming Language :: Python :: 3.12",
35
+ "Programming Language :: Python :: 3.13",
36
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
37
+ "Topic :: Scientific/Engineering :: Image Recognition",
38
+ "Typing :: Typed",
39
+ ]
40
+ dependencies = [
41
+ "requests>=2.31,<3",
42
+ ]
43
+
44
+ [project.optional-dependencies]
45
+ databricks = [
46
+ "databricks-sdk>=0.30",
47
+ ]
48
+ dev = [
49
+ "pytest>=8",
50
+ "responses>=0.25",
51
+ "build>=1.2",
52
+ "twine>=6.1",
53
+ ]
54
+
55
+ [project.urls]
56
+ Homepage = "https://dataclinic.ai"
57
+ "About DataClinic" = "https://dataclinic.ai/en/data-clinic/about"
58
+ Company = "https://www.pebblous.ai"
59
+
60
+ [tool.hatch.version]
61
+ path = "src/pebblous_dataclinic_gw_databricks/_version.py"
62
+
63
+ [tool.hatch.build.targets.wheel]
64
+ packages = ["src/pebblous_dataclinic_gw_databricks"]
65
+
66
+ [tool.hatch.build.targets.sdist]
67
+ include = [
68
+ "src/pebblous_dataclinic_gw_databricks",
69
+ "tests",
70
+ "README.md",
71
+ "CHANGELOG.md",
72
+ "LICENSE",
73
+ "pyproject.toml",
74
+ ]
75
+
76
+ [tool.pytest.ini_options]
77
+ testpaths = ["tests"]