pebblous-dataclinic-gw-databricks 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pebblous-dataclinic-gw-databricks might be problematic. Click here for more details.
- pebblous_dataclinic_gw_databricks-0.1.0/.gitignore +7 -0
- pebblous_dataclinic_gw_databricks-0.1.0/CHANGELOG.md +12 -0
- pebblous_dataclinic_gw_databricks-0.1.0/LICENSE +24 -0
- pebblous_dataclinic_gw_databricks-0.1.0/PKG-INFO +287 -0
- pebblous_dataclinic_gw_databricks-0.1.0/README.md +251 -0
- pebblous_dataclinic_gw_databricks-0.1.0/pyproject.toml +77 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/__init__.py +59 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/_version.py +1 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/auth.py +38 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/client.py +461 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/context.py +98 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/datasets.py +49 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/exceptions.py +105 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/logs.py +58 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/models.py +337 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/py.typed +0 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/results.py +54 -0
- pebblous_dataclinic_gw_databricks-0.1.0/src/pebblous_dataclinic_gw_databricks/transport.py +185 -0
- pebblous_dataclinic_gw_databricks-0.1.0/tests/conftest.py +55 -0
- pebblous_dataclinic_gw_databricks-0.1.0/tests/test_auth.py +138 -0
- pebblous_dataclinic_gw_databricks-0.1.0/tests/test_client.py +244 -0
- pebblous_dataclinic_gw_databricks-0.1.0/tests/test_context_results.py +132 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 — 2026-11-18
|
|
4
|
+
|
|
5
|
+
First release.
|
|
6
|
+
|
|
7
|
+
- `DataClinicGateway` with `from_databricks()` (Sharing Identifier, workspace ID and API key secret detection).
|
|
8
|
+
- DataClinic membership: `verify_member()`, `login()`, `logout()`, `member_session`. `analyze()` attaches `X-DataClinic-Member`.
|
|
9
|
+
- Datasets: `list_datasets()`, `display_datasets()`, `download_dataset()`, `ensure_dataset()`.
|
|
10
|
+
- Analysis: `analyze()`, `analysis_status()`, `analysis_logs()`, `wait()` with incremental logs, `stream_logs()` (SSE).
|
|
11
|
+
- Delta Sharing: `register_sharing()`, `analysis_runs()`, `analysis_results()`.
|
|
12
|
+
- Exceptions per Gateway status, no redirects, network retries that reuse the same `request_id`.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Copyright (c) 2026 Pebblous Inc. All rights reserved.
|
|
2
|
+
|
|
3
|
+
pebblous-dataclinic-gw-databricks ("the Software") is proprietary software of Pebblous Inc.
|
|
4
|
+
|
|
5
|
+
Permission is granted to download, install and run the Software, without modification, solely to
|
|
6
|
+
access the Pebblous DataClinic service from your own Databricks workspace or Python environment,
|
|
7
|
+
subject to the Pebblous DataClinic terms of service.
|
|
8
|
+
|
|
9
|
+
You may not, without prior written permission from Pebblous Inc.:
|
|
10
|
+
|
|
11
|
+
1. copy, modify, merge, translate or create derivative works of the Software, except as needed
|
|
12
|
+
to run it as permitted above;
|
|
13
|
+
2. distribute, sublicense, sell, rent or otherwise make the Software available to third parties,
|
|
14
|
+
other than through the official Python Package Index distribution;
|
|
15
|
+
3. use the Software to access any service other than the Pebblous DataClinic service, or to build
|
|
16
|
+
a competing product or service.
|
|
17
|
+
|
|
18
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
|
19
|
+
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
20
|
+
NONINFRINGEMENT. IN NO EVENT SHALL PEBBLOUS INC. BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
21
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
22
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
23
|
+
|
|
24
|
+
Contact: dev@pebblous.ai
|
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pebblous-dataclinic-gw-databricks
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Run Pebblous DataClinic image-data quality analysis from a Databricks notebook through the DataClinic Gateway.
|
|
5
|
+
Project-URL: Homepage, https://dataclinic.ai
|
|
6
|
+
Project-URL: About DataClinic, https://dataclinic.ai/en/data-clinic/about
|
|
7
|
+
Project-URL: Company, https://www.pebblous.ai
|
|
8
|
+
Author-email: Pebblous <dev@pebblous.ai>
|
|
9
|
+
Maintainer-email: Pebblous <dev@pebblous.ai>
|
|
10
|
+
License-Expression: LicenseRef-Pebblous-Proprietary
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: data-quality,databricks,dataclinic,delta-sharing,embedding,huggingface,image-data,pebblous
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Image Recognition
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: requests<3,>=2.31
|
|
28
|
+
Provides-Extra: databricks
|
|
29
|
+
Requires-Dist: databricks-sdk>=0.30; extra == 'databricks'
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
33
|
+
Requires-Dist: responses>=0.25; extra == 'dev'
|
|
34
|
+
Requires-Dist: twine>=6.1; extra == 'dev'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# pebblous-dataclinic-gw-databricks
|
|
38
|
+
|
|
39
|
+
Run [Pebblous DataClinic](https://dataclinic.ai) image-data quality analysis from a Databricks notebook.
|
|
40
|
+
|
|
41
|
+
This package is the client for the **Pebblous DataClinic Gateway**. Your notebook asks the Gateway to
|
|
42
|
+
prepare a dataset and run the DataClinic analyzer on Pebblous GPUs. The results (image embeddings,
|
|
43
|
+
density and k-nearest neighbours) come back to your workspace through Delta Sharing. You never
|
|
44
|
+
handle Pebblous source code or credentials, and your notebook never handles raw HTTP.
|
|
45
|
+
|
|
46
|
+
```text
|
|
47
|
+
Your Databricks notebook ──► Pebblous Gateway ──► Databricks Jobs (download, GPU analysis)
|
|
48
|
+
▲ │
|
|
49
|
+
└────────────── Delta Sharing (results) ◄─────────┘
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
- **Installs one dependency:** `requests`. `databricks-sdk` is optional.
|
|
53
|
+
- **Collects Databricks context for you:** it reads your Metastore Sharing Identifier and workspace ID automatically.
|
|
54
|
+
- **Signs you in once:** you log in with your dataclinic.ai account, and the member token stays in memory only.
|
|
55
|
+
- **Handles waiting and logs:** it polls run status and shows only the new part of each log.
|
|
56
|
+
- **Raises clear exceptions:** each Gateway error maps to an exception class you can catch.
|
|
57
|
+
|
|
58
|
+
> **Status:** alpha (0.1). The API may change before 1.0.
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## Requirements
|
|
63
|
+
|
|
64
|
+
| | |
|
|
65
|
+
| --- | --- |
|
|
66
|
+
| Python | 3.10 or newer |
|
|
67
|
+
| Runtime | Databricks notebook (Unity Catalog enabled) or any Python 3.10+ |
|
|
68
|
+
| Access | A Gateway URL and API key from Pebblous, and a [dataclinic.ai](https://dataclinic.ai) account (the Free plan works) |
|
|
69
|
+
|
|
70
|
+
## Install
|
|
71
|
+
|
|
72
|
+
In a Databricks notebook:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
%pip install "pebblous-dataclinic-gw-databricks[databricks]"
|
|
76
|
+
dbutils.library.restartPython()
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Anywhere else:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install pebblous-dataclinic-gw-databricks
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Quick start (Databricks notebook)
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from pebblous_dataclinic_gw_databricks import DataClinicGateway
|
|
89
|
+
|
|
90
|
+
# 1. Connect. The Sharing Identifier and workspace ID are detected from the notebook.
|
|
91
|
+
# The API key is read from the secret dataclinic/api_key unless you pass api_key=...
|
|
92
|
+
clinic = DataClinicGateway.from_databricks(gateway_url="https://<your-gateway-host>")
|
|
93
|
+
clinic.health()
|
|
94
|
+
|
|
95
|
+
# 2. Sign in with your dataclinic.ai account. The password is asked with a hidden prompt.
|
|
96
|
+
clinic.login()
|
|
97
|
+
|
|
98
|
+
# 3. Choose a dataset and make sure it is downloaded (no GPU used here).
|
|
99
|
+
clinic.display_datasets(refresh=True)
|
|
100
|
+
clinic.ensure_dataset("beans", max_images=300, show_logs=True)
|
|
101
|
+
|
|
102
|
+
# 4. Run the GPU analysis and follow its log.
|
|
103
|
+
run = clinic.analyze(preset="beans", max_images=300, wait=True, show_logs=True)
|
|
104
|
+
print(run.analysis_id, run.status)
|
|
105
|
+
|
|
106
|
+
# 5. Receive the results through Delta Sharing.
|
|
107
|
+
clinic.register_sharing()
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Then create the shared catalog once, in SQL:
|
|
111
|
+
|
|
112
|
+
```sql
|
|
113
|
+
CREATE CATALOG IF NOT EXISTS pebblous_dataclinic_shared
|
|
114
|
+
USING SHARE Pebblous.dataclinic_demo_share;
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
And read your results:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
display(clinic.analysis_runs())
|
|
121
|
+
display(clinic.analysis_results(run.analysis_id))
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## DataClinic membership
|
|
125
|
+
|
|
126
|
+
The Gateway can require a [dataclinic.ai](https://dataclinic.ai) account before it runs the GPU analysis.
|
|
127
|
+
Every active member qualifies, including the Free plan.
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
member = clinic.login() # prompts for email and password
|
|
131
|
+
member = clinic.login("me@company.com") # prompts for the password only
|
|
132
|
+
member = clinic.verify_member(email, password) # non-interactive
|
|
133
|
+
|
|
134
|
+
clinic.member_session # MemberSession(member_id=..., plan=..., expires_at=...) or None
|
|
135
|
+
clinic.logout() # forget the token
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
| Method | What it does |
|
|
139
|
+
| --- | --- |
|
|
140
|
+
| `verify_member(email, password)` | Calls the Gateway member API (`POST /api/v1/members/verify`) and keeps the returned member token in memory |
|
|
141
|
+
| `login(email=None)` | Notebook login. It uses the secrets `dataclinic/member_email` and `dataclinic/member_password` if both exist. Otherwise it asks for the email and reads the password with `getpass` |
|
|
142
|
+
| `member_session` | The current `MemberSession`, kept in memory only. It is `None` before login, after logout, and after expiry |
|
|
143
|
+
| `analyze()` | Adds the `X-DataClinic-Member` header automatically when you are signed in |
|
|
144
|
+
| `logout()` | Removes the member token from memory |
|
|
145
|
+
|
|
146
|
+
- The password is sent once to the Gateway and never stored. The SDK keeps no copy.
|
|
147
|
+
- The token is left out of `repr()`, printing and exception messages.
|
|
148
|
+
- A token lasts about one hour. After it expires, `analyze()` raises `MemberAuthRequiredError`; call `login()` again.
|
|
149
|
+
- If the Gateway does not use membership checks, `login()` returns `None` and analysis works without it.
|
|
150
|
+
- If the Gateway URL is plain `http://` and not localhost, the SDK warns with `InsecureGatewayWarning` before it sends the password.
|
|
151
|
+
|
|
152
|
+
To skip the prompt in scheduled jobs, store the credentials as Databricks secrets:
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
databricks secrets put-secret dataclinic member_email
|
|
156
|
+
databricks secrets put-secret dataclinic member_password
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## Datasets
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
catalog = clinic.list_datasets(refresh=True)
|
|
163
|
+
catalog.volume_datasets # ['demo'] - folders already in the Pebblous Volume
|
|
164
|
+
catalog.presets # Hugging Face presets with cache state
|
|
165
|
+
beans = catalog.get("beans")
|
|
166
|
+
beans.downloaded_images # images ready for analysis now
|
|
167
|
+
beans.ready_for(300) # True if 300 images can be analysed without a download
|
|
168
|
+
catalog.analyzable() # presets you can analyse right away
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Hugging Face datasets are **downloaded first, analysed second**. The GPU job never downloads images.
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
download = clinic.ensure_dataset("mnist", max_images=500) # waits by default
|
|
175
|
+
download.status # READY or SUCCEEDED
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
`download_dataset()` returns `status="READY"` and `run_id=None` when the Pebblous Volume already
|
|
179
|
+
holds enough images. That counts as success, so no job is started.
|
|
180
|
+
|
|
181
|
+
## Analysis
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
run = clinic.analyze() # the Pebblous "demo" folder
|
|
185
|
+
run = clinic.analyze(preset="beans", max_images=300) # a downloaded preset
|
|
186
|
+
run.run_id # Databricks run number, used for status and logs
|
|
187
|
+
run.analysis_id # key of your rows in the shared result tables
|
|
188
|
+
|
|
189
|
+
clinic.wait(run.run_id, show_logs=True) # resume watching at any time
|
|
190
|
+
clinic.analysis_status(run.run_id)
|
|
191
|
+
clinic.analysis_logs(run.run_id).text
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Live log stream (server-sent events):
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
for event in clinic.stream_logs(run.run_id):
|
|
198
|
+
if event.type == "log":
|
|
199
|
+
print(event.text, end="")
|
|
200
|
+
elif event.type == "end":
|
|
201
|
+
print("\nfinished:", event.status)
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
A `request_id` (UUID) is generated for every request and reused when the SDK retries after a network
|
|
205
|
+
error, so a retry never starts a second job. Pass your own `request_id=` to make a cell safe to re-run.
|
|
206
|
+
|
|
207
|
+
## Results
|
|
208
|
+
|
|
209
|
+
`analysis_runs()` and `analysis_results()` read the shared views with Spark. They make no Gateway call.
|
|
210
|
+
|
|
211
|
+
| View | Content |
|
|
212
|
+
| --- | --- |
|
|
213
|
+
| `clinic_runs` | One row per analysis: dataset, image count, status, duration |
|
|
214
|
+
| `clinic_analysis` | One row per image: `embedding`, `norm`, `density`, `nearest_id`, `nearest_distance`, `knn` |
|
|
215
|
+
|
|
216
|
+
The views only show rows that belong to your Metastore. Pebblous filters them on the share side
|
|
217
|
+
with `CURRENT_RECIPIENT`.
|
|
218
|
+
|
|
219
|
+
## Plain Python
|
|
220
|
+
|
|
221
|
+
Outside Databricks, pass the context yourself:
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
clinic = DataClinicGateway(
|
|
225
|
+
gateway_url="https://<your-gateway-host>",
|
|
226
|
+
api_key="...",
|
|
227
|
+
sharing_identifier="aws:us-west-2:00000000-0000-0000-0000-000000000000",
|
|
228
|
+
workspace_id=1234567890123456,
|
|
229
|
+
)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Everything except `analysis_runs()` and `analysis_results()` works the same way.
|
|
233
|
+
|
|
234
|
+
## Errors
|
|
235
|
+
|
|
236
|
+
| Exception | HTTP | Typical cause |
|
|
237
|
+
| --- | --- | --- |
|
|
238
|
+
| `InvalidRequestError` | 400, 422 | Unknown preset, image count over the limit |
|
|
239
|
+
| `DatasetNotDownloadedError` | 400 | `analyze()` before `ensure_dataset()` |
|
|
240
|
+
| `AuthenticationError` | 401 | Wrong API key, or wrong dataclinic.ai email/password |
|
|
241
|
+
| `MemberAuthRequiredError` | 401 | Membership required, or the session expired. Call `login()` |
|
|
242
|
+
| `AuthorizationError` | 403 | Not a dataclinic.ai member, a disabled account, or a token issued for another Metastore |
|
|
243
|
+
| `ResourceNotFoundError` | 404 | Unknown run, or a run started by another Metastore |
|
|
244
|
+
| `RequestConflictError` | 409 | `request_id` reused for a different request |
|
|
245
|
+
| `RateLimitError` | 429 | Another run is active, or too many login attempts. Not retried automatically |
|
|
246
|
+
| `GatewayUnavailableError` | 502, 503 | Gateway cannot reach Databricks, Hugging Face or dataclinic.ai |
|
|
247
|
+
| `GatewayConnectionError` | – | Network problem between you and the Gateway |
|
|
248
|
+
| `RunFailedError` | – | `wait()` saw `FAILED` or `CANCELED`. The final run is in `.run` |
|
|
249
|
+
| `WaitTimeoutError` | – | `wait()` stopped watching. The run continues; call `wait()` again |
|
|
250
|
+
|
|
251
|
+
All of them inherit from `DataClinicError`, which has `status_code` and `detail` attributes.
|
|
252
|
+
|
|
253
|
+
## Security notes
|
|
254
|
+
|
|
255
|
+
- The package contains no Pebblous credentials. Keep your Gateway API key in a Databricks secret.
|
|
256
|
+
- The SDK does not follow HTTP redirects, so your API key is never sent to another host.
|
|
257
|
+
- The API key, password and member token never appear in `repr()`, logs or exceptions.
|
|
258
|
+
- Use an `https://` Gateway URL.
|
|
259
|
+
|
|
260
|
+
## API summary
|
|
261
|
+
|
|
262
|
+
| Method | Gateway endpoint |
|
|
263
|
+
| --- | --- |
|
|
264
|
+
| `health()` | `GET /health` |
|
|
265
|
+
| `list_datasets(refresh=False)`, `display_datasets()` | `GET /api/v1/datasets` |
|
|
266
|
+
| `verify_member(email, password)`, `login()` | `POST /api/v1/members/verify` |
|
|
267
|
+
| `logout()`, `member_session` | (local) |
|
|
268
|
+
| `download_dataset(...)`, `ensure_dataset(...)` | `POST /api/v1/downloads` |
|
|
269
|
+
| `download_status(run_id)`, `download_logs(run_id)` | `GET /api/v1/downloads/{run_id}[/logs]` |
|
|
270
|
+
| `analyze(...)` | `POST /api/v1/analyses` |
|
|
271
|
+
| `analysis_status(run_id)`, `analysis_logs(run_id)` | `GET /api/v1/analyses/{run_id}[/logs]` |
|
|
272
|
+
| `wait(run_id, kind)` | status and log polling |
|
|
273
|
+
| `stream_logs(run_id, kind)` | `GET .../logs?follow=true` (SSE) |
|
|
274
|
+
| `register_sharing()` | `POST /api/v1/sharing/register` |
|
|
275
|
+
| `analysis_runs()`, `analysis_results(analysis_id)` | Spark, shared catalog |
|
|
276
|
+
|
|
277
|
+
## Support
|
|
278
|
+
|
|
279
|
+
- DataClinic: <https://dataclinic.ai>
|
|
280
|
+
- Pebblous: <https://www.pebblous.ai>
|
|
281
|
+
- Email: dev@pebblous.ai
|
|
282
|
+
|
|
283
|
+
## License
|
|
284
|
+
|
|
285
|
+
Proprietary. Copyright (c) 2026 Pebblous Inc. You may install and run this package, unmodified, to use
|
|
286
|
+
the Pebblous DataClinic service, subject to the DataClinic terms of service. See the `LICENSE` file in
|
|
287
|
+
the distribution for details.
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
# pebblous-dataclinic-gw-databricks
|
|
2
|
+
|
|
3
|
+
Run [Pebblous DataClinic](https://dataclinic.ai) image-data quality analysis from a Databricks notebook.
|
|
4
|
+
|
|
5
|
+
This package is the client for the **Pebblous DataClinic Gateway**. Your notebook asks the Gateway to
|
|
6
|
+
prepare a dataset and run the DataClinic analyzer on Pebblous GPUs. The results (image embeddings,
|
|
7
|
+
density and k-nearest neighbours) come back to your workspace through Delta Sharing. You never
|
|
8
|
+
handle Pebblous source code or credentials, and your notebook never handles raw HTTP.
|
|
9
|
+
|
|
10
|
+
```text
|
|
11
|
+
Your Databricks notebook ──► Pebblous Gateway ──► Databricks Jobs (download, GPU analysis)
|
|
12
|
+
▲ │
|
|
13
|
+
└────────────── Delta Sharing (results) ◄─────────┘
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
- **Installs one dependency:** `requests`. `databricks-sdk` is optional.
|
|
17
|
+
- **Collects Databricks context for you:** it reads your Metastore Sharing Identifier and workspace ID automatically.
|
|
18
|
+
- **Signs you in once:** you log in with your dataclinic.ai account, and the member token stays in memory only.
|
|
19
|
+
- **Handles waiting and logs:** it polls run status and shows only the new part of each log.
|
|
20
|
+
- **Raises clear exceptions:** each Gateway error maps to an exception class you can catch.
|
|
21
|
+
|
|
22
|
+
> **Status:** alpha (0.1). The API may change before 1.0.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## Requirements
|
|
27
|
+
|
|
28
|
+
| | |
|
|
29
|
+
| --- | --- |
|
|
30
|
+
| Python | 3.10 or newer |
|
|
31
|
+
| Runtime | Databricks notebook (Unity Catalog enabled) or any Python 3.10+ |
|
|
32
|
+
| Access | A Gateway URL and API key from Pebblous, and a [dataclinic.ai](https://dataclinic.ai) account (the Free plan works) |
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
In a Databricks notebook:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
%pip install "pebblous-dataclinic-gw-databricks[databricks]"
|
|
40
|
+
dbutils.library.restartPython()
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Anywhere else:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install pebblous-dataclinic-gw-databricks
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Quick start (Databricks notebook)
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from pebblous_dataclinic_gw_databricks import DataClinicGateway
|
|
53
|
+
|
|
54
|
+
# 1. Connect. The Sharing Identifier and workspace ID are detected from the notebook.
|
|
55
|
+
# The API key is read from the secret dataclinic/api_key unless you pass api_key=...
|
|
56
|
+
clinic = DataClinicGateway.from_databricks(gateway_url="https://<your-gateway-host>")
|
|
57
|
+
clinic.health()
|
|
58
|
+
|
|
59
|
+
# 2. Sign in with your dataclinic.ai account. The password is asked with a hidden prompt.
|
|
60
|
+
clinic.login()
|
|
61
|
+
|
|
62
|
+
# 3. Choose a dataset and make sure it is downloaded (no GPU used here).
|
|
63
|
+
clinic.display_datasets(refresh=True)
|
|
64
|
+
clinic.ensure_dataset("beans", max_images=300, show_logs=True)
|
|
65
|
+
|
|
66
|
+
# 4. Run the GPU analysis and follow its log.
|
|
67
|
+
run = clinic.analyze(preset="beans", max_images=300, wait=True, show_logs=True)
|
|
68
|
+
print(run.analysis_id, run.status)
|
|
69
|
+
|
|
70
|
+
# 5. Receive the results through Delta Sharing.
|
|
71
|
+
clinic.register_sharing()
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Then create the shared catalog once, in SQL:
|
|
75
|
+
|
|
76
|
+
```sql
|
|
77
|
+
CREATE CATALOG IF NOT EXISTS pebblous_dataclinic_shared
|
|
78
|
+
USING SHARE Pebblous.dataclinic_demo_share;
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
And read your results:
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
display(clinic.analysis_runs())
|
|
85
|
+
display(clinic.analysis_results(run.analysis_id))
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## DataClinic membership
|
|
89
|
+
|
|
90
|
+
The Gateway can require a [dataclinic.ai](https://dataclinic.ai) account before it runs the GPU analysis.
|
|
91
|
+
Every active member qualifies, including the Free plan.
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
member = clinic.login() # prompts for email and password
|
|
95
|
+
member = clinic.login("me@company.com") # prompts for the password only
|
|
96
|
+
member = clinic.verify_member(email, password) # non-interactive
|
|
97
|
+
|
|
98
|
+
clinic.member_session # MemberSession(member_id=..., plan=..., expires_at=...) or None
|
|
99
|
+
clinic.logout() # forget the token
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
| Method | What it does |
|
|
103
|
+
| --- | --- |
|
|
104
|
+
| `verify_member(email, password)` | Calls the Gateway member API (`POST /api/v1/members/verify`) and keeps the returned member token in memory |
|
|
105
|
+
| `login(email=None)` | Notebook login. It uses the secrets `dataclinic/member_email` and `dataclinic/member_password` if both exist. Otherwise it asks for the email and reads the password with `getpass` |
|
|
106
|
+
| `member_session` | The current `MemberSession`, kept in memory only. It is `None` before login, after logout, and after expiry |
|
|
107
|
+
| `analyze()` | Adds the `X-DataClinic-Member` header automatically when you are signed in |
|
|
108
|
+
| `logout()` | Removes the member token from memory |
|
|
109
|
+
|
|
110
|
+
- The password is sent once to the Gateway and never stored. The SDK keeps no copy.
|
|
111
|
+
- The token is left out of `repr()`, printing and exception messages.
|
|
112
|
+
- A token lasts about one hour. After it expires, `analyze()` raises `MemberAuthRequiredError`; call `login()` again.
|
|
113
|
+
- If the Gateway does not use membership checks, `login()` returns `None` and analysis works without it.
|
|
114
|
+
- If the Gateway URL is plain `http://` and not localhost, the SDK warns with `InsecureGatewayWarning` before it sends the password.
|
|
115
|
+
|
|
116
|
+
To skip the prompt in scheduled jobs, store the credentials as Databricks secrets:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
databricks secrets put-secret dataclinic member_email
|
|
120
|
+
databricks secrets put-secret dataclinic member_password
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Datasets
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
catalog = clinic.list_datasets(refresh=True)
|
|
127
|
+
catalog.volume_datasets # ['demo'] - folders already in the Pebblous Volume
|
|
128
|
+
catalog.presets # Hugging Face presets with cache state
|
|
129
|
+
beans = catalog.get("beans")
|
|
130
|
+
beans.downloaded_images # images ready for analysis now
|
|
131
|
+
beans.ready_for(300) # True if 300 images can be analysed without a download
|
|
132
|
+
catalog.analyzable() # presets you can analyse right away
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Hugging Face datasets are **downloaded first, analysed second**. The GPU job never downloads images.
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
download = clinic.ensure_dataset("mnist", max_images=500) # waits by default
|
|
139
|
+
download.status # READY or SUCCEEDED
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
`download_dataset()` returns `status="READY"` and `run_id=None` when the Pebblous Volume already
|
|
143
|
+
holds enough images. That counts as success, so no job is started.
|
|
144
|
+
|
|
145
|
+
## Analysis
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
run = clinic.analyze() # the Pebblous "demo" folder
|
|
149
|
+
run = clinic.analyze(preset="beans", max_images=300) # a downloaded preset
|
|
150
|
+
run.run_id # Databricks run number, used for status and logs
|
|
151
|
+
run.analysis_id # key of your rows in the shared result tables
|
|
152
|
+
|
|
153
|
+
clinic.wait(run.run_id, show_logs=True) # resume watching at any time
|
|
154
|
+
clinic.analysis_status(run.run_id)
|
|
155
|
+
clinic.analysis_logs(run.run_id).text
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Live log stream (server-sent events):
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
for event in clinic.stream_logs(run.run_id):
|
|
162
|
+
if event.type == "log":
|
|
163
|
+
print(event.text, end="")
|
|
164
|
+
elif event.type == "end":
|
|
165
|
+
print("\nfinished:", event.status)
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
A `request_id` (UUID) is generated for every request and reused when the SDK retries after a network
|
|
169
|
+
error, so a retry never starts a second job. Pass your own `request_id=` to make a cell safe to re-run.
|
|
170
|
+
|
|
171
|
+
## Results
|
|
172
|
+
|
|
173
|
+
`analysis_runs()` and `analysis_results()` read the shared views with Spark. They make no Gateway call.
|
|
174
|
+
|
|
175
|
+
| View | Content |
|
|
176
|
+
| --- | --- |
|
|
177
|
+
| `clinic_runs` | One row per analysis: dataset, image count, status, duration |
|
|
178
|
+
| `clinic_analysis` | One row per image: `embedding`, `norm`, `density`, `nearest_id`, `nearest_distance`, `knn` |
|
|
179
|
+
|
|
180
|
+
The views only show rows that belong to your Metastore. Pebblous filters them on the share side
|
|
181
|
+
with `CURRENT_RECIPIENT`.
|
|
182
|
+
|
|
183
|
+
## Plain Python
|
|
184
|
+
|
|
185
|
+
Outside Databricks, pass the context yourself:
|
|
186
|
+
|
|
187
|
+
```python
|
|
188
|
+
clinic = DataClinicGateway(
|
|
189
|
+
gateway_url="https://<your-gateway-host>",
|
|
190
|
+
api_key="...",
|
|
191
|
+
sharing_identifier="aws:us-west-2:00000000-0000-0000-0000-000000000000",
|
|
192
|
+
workspace_id=1234567890123456,
|
|
193
|
+
)
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Everything except `analysis_runs()` and `analysis_results()` works the same way.
|
|
197
|
+
|
|
198
|
+
## Errors
|
|
199
|
+
|
|
200
|
+
| Exception | HTTP | Typical cause |
|
|
201
|
+
| --- | --- | --- |
|
|
202
|
+
| `InvalidRequestError` | 400, 422 | Unknown preset, image count over the limit |
|
|
203
|
+
| `DatasetNotDownloadedError` | 400 | `analyze()` before `ensure_dataset()` |
|
|
204
|
+
| `AuthenticationError` | 401 | Wrong API key, or wrong dataclinic.ai email/password |
|
|
205
|
+
| `MemberAuthRequiredError` | 401 | Membership required, or the session expired. Call `login()` |
|
|
206
|
+
| `AuthorizationError` | 403 | Not a dataclinic.ai member, a disabled account, or a token issued for another Metastore |
|
|
207
|
+
| `ResourceNotFoundError` | 404 | Unknown run, or a run started by another Metastore |
|
|
208
|
+
| `RequestConflictError` | 409 | `request_id` reused for a different request |
|
|
209
|
+
| `RateLimitError` | 429 | Another run is active, or too many login attempts. Not retried automatically |
|
|
210
|
+
| `GatewayUnavailableError` | 502, 503 | Gateway cannot reach Databricks, Hugging Face or dataclinic.ai |
|
|
211
|
+
| `GatewayConnectionError` | – | Network problem between you and the Gateway |
|
|
212
|
+
| `RunFailedError` | – | `wait()` saw `FAILED` or `CANCELED`. The final run is in `.run` |
|
|
213
|
+
| `WaitTimeoutError` | – | `wait()` stopped watching. The run continues; call `wait()` again |
|
|
214
|
+
|
|
215
|
+
All of them inherit from `DataClinicError`, which has `status_code` and `detail` attributes.
|
|
216
|
+
|
|
217
|
+
## Security notes
|
|
218
|
+
|
|
219
|
+
- The package contains no Pebblous credentials. Keep your Gateway API key in a Databricks secret.
|
|
220
|
+
- The SDK does not follow HTTP redirects, so your API key is never sent to another host.
|
|
221
|
+
- The API key, password and member token never appear in `repr()`, logs or exceptions.
|
|
222
|
+
- Use an `https://` Gateway URL.
|
|
223
|
+
|
|
224
|
+
## API summary
|
|
225
|
+
|
|
226
|
+
| Method | Gateway endpoint |
|
|
227
|
+
| --- | --- |
|
|
228
|
+
| `health()` | `GET /health` |
|
|
229
|
+
| `list_datasets(refresh=False)`, `display_datasets()` | `GET /api/v1/datasets` |
|
|
230
|
+
| `verify_member(email, password)`, `login()` | `POST /api/v1/members/verify` |
|
|
231
|
+
| `logout()`, `member_session` | (local) |
|
|
232
|
+
| `download_dataset(...)`, `ensure_dataset(...)` | `POST /api/v1/downloads` |
|
|
233
|
+
| `download_status(run_id)`, `download_logs(run_id)` | `GET /api/v1/downloads/{run_id}[/logs]` |
|
|
234
|
+
| `analyze(...)` | `POST /api/v1/analyses` |
|
|
235
|
+
| `analysis_status(run_id)`, `analysis_logs(run_id)` | `GET /api/v1/analyses/{run_id}[/logs]` |
|
|
236
|
+
| `wait(run_id, kind)` | status and log polling |
|
|
237
|
+
| `stream_logs(run_id, kind)` | `GET .../logs?follow=true` (SSE) |
|
|
238
|
+
| `register_sharing()` | `POST /api/v1/sharing/register` |
|
|
239
|
+
| `analysis_runs()`, `analysis_results(analysis_id)` | Spark, shared catalog |
|
|
240
|
+
|
|
241
|
+
## Support
|
|
242
|
+
|
|
243
|
+
- DataClinic: <https://dataclinic.ai>
|
|
244
|
+
- Pebblous: <https://www.pebblous.ai>
|
|
245
|
+
- Email: dev@pebblous.ai
|
|
246
|
+
|
|
247
|
+
## License
|
|
248
|
+
|
|
249
|
+
Proprietary. Copyright (c) 2026 Pebblous Inc. You may install and run this package, unmodified, to use
|
|
250
|
+
the Pebblous DataClinic service, subject to the DataClinic terms of service. See the `LICENSE` file in
|
|
251
|
+
the distribution for details.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pebblous-dataclinic-gw-databricks"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Run Pebblous DataClinic image-data quality analysis from a Databricks notebook through the DataClinic Gateway."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "LicenseRef-Pebblous-Proprietary"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Pebblous", email = "dev@pebblous.ai" }]
|
|
14
|
+
maintainers = [{ name = "Pebblous", email = "dev@pebblous.ai" }]
|
|
15
|
+
keywords = [
|
|
16
|
+
"databricks",
|
|
17
|
+
"dataclinic",
|
|
18
|
+
"pebblous",
|
|
19
|
+
"data-quality",
|
|
20
|
+
"image-data",
|
|
21
|
+
"embedding",
|
|
22
|
+
"delta-sharing",
|
|
23
|
+
"huggingface",
|
|
24
|
+
]
|
|
25
|
+
classifiers = [
|
|
26
|
+
"Development Status :: 3 - Alpha",
|
|
27
|
+
"Intended Audience :: Developers",
|
|
28
|
+
"Intended Audience :: Science/Research",
|
|
29
|
+
"Operating System :: OS Independent",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
32
|
+
"Programming Language :: Python :: 3.10",
|
|
33
|
+
"Programming Language :: Python :: 3.11",
|
|
34
|
+
"Programming Language :: Python :: 3.12",
|
|
35
|
+
"Programming Language :: Python :: 3.13",
|
|
36
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
37
|
+
"Topic :: Scientific/Engineering :: Image Recognition",
|
|
38
|
+
"Typing :: Typed",
|
|
39
|
+
]
|
|
40
|
+
dependencies = [
|
|
41
|
+
"requests>=2.31,<3",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.optional-dependencies]
|
|
45
|
+
databricks = [
|
|
46
|
+
"databricks-sdk>=0.30",
|
|
47
|
+
]
|
|
48
|
+
dev = [
|
|
49
|
+
"pytest>=8",
|
|
50
|
+
"responses>=0.25",
|
|
51
|
+
"build>=1.2",
|
|
52
|
+
"twine>=6.1",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
[project.urls]
|
|
56
|
+
Homepage = "https://dataclinic.ai"
|
|
57
|
+
"About DataClinic" = "https://dataclinic.ai/en/data-clinic/about"
|
|
58
|
+
Company = "https://www.pebblous.ai"
|
|
59
|
+
|
|
60
|
+
[tool.hatch.version]
|
|
61
|
+
path = "src/pebblous_dataclinic_gw_databricks/_version.py"
|
|
62
|
+
|
|
63
|
+
[tool.hatch.build.targets.wheel]
|
|
64
|
+
packages = ["src/pebblous_dataclinic_gw_databricks"]
|
|
65
|
+
|
|
66
|
+
[tool.hatch.build.targets.sdist]
|
|
67
|
+
include = [
|
|
68
|
+
"src/pebblous_dataclinic_gw_databricks",
|
|
69
|
+
"tests",
|
|
70
|
+
"README.md",
|
|
71
|
+
"CHANGELOG.md",
|
|
72
|
+
"LICENSE",
|
|
73
|
+
"pyproject.toml",
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
[tool.pytest.ini_options]
|
|
77
|
+
testpaths = ["tests"]
|