marpledata 3.5.0__tar.gz → 3.6.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/AGENTS.md +3 -1
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/CHANGELOG.md +22 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/CONTRIBUTING.md +2 -3
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/PKG-INFO +42 -1
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/README.md +41 -0
- marpledata-3.6.0.dev2/RELEASING.md +13 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/db.rst +4 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.DB.rst +7 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.DataStream.rst +5 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.Dataset.rst +4 -0
- marpledata-3.6.0.dev2/docs/api/marple.db.SandboxJob.rst +43 -0
- marpledata-3.6.0.dev2/docs/api/marple.db.SandboxJobStatus.rst +28 -0
- marpledata-3.6.0.dev2/docs/api/marple.db.Script.rst +41 -0
- marpledata-3.6.0.dev2/docs/api/marple.db.ScriptVersion.rst +21 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/getting-started.rst +7 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/tutorials.rst +41 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/pyproject.toml +6 -1
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/__init__.py +1 -1
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/__init__.py +105 -1
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/dataset.py +166 -21
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/datastream.py +112 -16
- marpledata-3.6.0.dev2/src/marple/db/script.py +182 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/signal.py +13 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/signal_upload.py +1 -1
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/utils.py +26 -15
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/conftest.py +6 -10
- marpledata-3.6.0.dev2/tests/support.py +60 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_db.py +111 -83
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_insight.py +3 -1
- marpledata-3.6.0.dev2/tests/test_scripts.py +266 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_signal_upload.py +148 -108
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_sql.py +3 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/uv.lock +1 -1
- marpledata-3.5.0/RELEASING.md +0 -11
- marpledata-3.5.0/tests/support.py +0 -24
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.flake8 +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.gitignore +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.python-version +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/.gitignore +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/.lock +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/CACHEDIR.TAG +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/interpreter-v4/c2c2931c8a99aae1/588e1e7c128e2872.msgpack +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/sdists-v9/.gitignore +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/LICENSE +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/DEPLOYMENT.md +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/_static/custom.css +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/_static/favicon.png +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/_static/logo.png +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/insight.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.Insight.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.DatasetList.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.LAKE_ARROW_SCHEMA.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.SCHEMA.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.Signal.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.SignalUpload.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.SignalsAlreadyExistError.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/conf.py +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/index.rst +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/example.py +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/examples_race.csv +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/pytest.xml +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/constants.py +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/sql.py +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/insight.py +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/py.typed +0 -0
- {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_reliability.py +0 -0
|
@@ -15,6 +15,7 @@ This directory contains the Python SDK package published as `marpledata`.
|
|
|
15
15
|
## Commands
|
|
16
16
|
|
|
17
17
|
- Run tests with output: `uv run pytest -vs`
|
|
18
|
+
- Unit tests only: `uv run pytest -m "not integration"`
|
|
18
19
|
- Build docs: `uv run --group docs sphinx-build -b html docs docs/_build/html`
|
|
19
20
|
- Build package: `uv build`
|
|
20
21
|
- Fix formatting: `uv run isort src tests && uv run black src tests`
|
|
@@ -29,5 +30,6 @@ Run commands from `python/` unless a command explicitly says otherwise.
|
|
|
29
30
|
- Keep Python SDK tests in `python/tests/`.
|
|
30
31
|
- Integration tests run against live Marple services and may require
|
|
31
32
|
`MDB_TOKEN`, `MDB_URL`, `INSIGHT_TOKEN`, and `INSIGHT_URL`. Tests should skip
|
|
32
|
-
|
|
33
|
+
clearly when required credentials are missing. Mark live tests with
|
|
34
|
+
`pytest.mark.integration`.
|
|
33
35
|
- Release steps: `RELEASING.md`.
|
|
@@ -5,6 +5,28 @@ All notable changes to the Python SDK package `marpledata` will be documented in
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
|
|
12
|
+
- Exact signal names in `get_data` / `get_signal` / `get_signals` resolve via the dataset cache or `GET /datapool/{pool}/signal/{name}/id` instead of downloading the full datapool `signal_map`. Regex patterns still use the map.
|
|
13
|
+
- Optional `plugin_args` on `DataStream.push_file` (and the deprecated `DB.push_file` wrapper).
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- Processing Pipeline
|
|
18
|
+
- `DB.create_script` / `get_scripts` / `get_script` / `delete_script`
|
|
19
|
+
- `Script.update` / `duplicate` / `delete`, to manage stored `process(dataset)` scripts.
|
|
20
|
+
- `Dataset.run` / `DB.run_script` to execute a stored processing script against a dataset via a server-side sandbox job.
|
|
21
|
+
- `DataStream.scripts`, `DataStream.update`, and `DataStream.rerun_processing` (`DB.rerun_processing`) to edit a stream and set the script pipeline
|
|
22
|
+
- `Dataset.rerun_processing` and `Dataset.get_debug_messages`.
|
|
23
|
+
- `DB.delete_signals`, `Dataset.delete_signal` / `Dataset.delete_signals`, and `Signal.delete` to remove signals from a dataset.
|
|
24
|
+
- `Dataset.reingest` to reingest a dataset from its original uploaded file, with optional `plugin_args`.
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- Allow `add_signal` in post-processing
|
|
29
|
+
|
|
8
30
|
## [3.5.0] - 2026-08-20
|
|
9
31
|
|
|
10
32
|
### Added
|
|
@@ -22,11 +22,10 @@ uv run mypy --install-types --non-interactive
|
|
|
22
22
|
|
|
23
23
|
### Testing
|
|
24
24
|
|
|
25
|
-
The testing suite
|
|
26
|
-
Among other things it will create a stream, ingest a dataset, and export that dataset from Insight
|
|
27
|
-
Upload tests also exercise server and multipart upload flows, and create/delete temporary streams with a test prefix.
|
|
25
|
+
The testing suite includes local unit tests and integration tests against a live Marple DB / Insight deployment. Integration tests share one imported example dataset per session, use a small CSV for extra ingestions, and mark themselves with `pytest.mark.integration`.
|
|
28
26
|
|
|
29
27
|
```bash
|
|
28
|
+
uv run pytest -m "not integration" # local unit tests only
|
|
30
29
|
export MDB_TOKEN=...
|
|
31
30
|
export INSIGHT_TOKEN=...
|
|
32
31
|
# Optional, defaults to SaaS URLs:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: marpledata
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.6.0.dev2
|
|
4
4
|
Summary: Marple SDK for Python
|
|
5
5
|
Project-URL: Homepage, https://www.marpledata.com/
|
|
6
6
|
Project-URL: Documentation, https://marpledata.gitlab.io/marple-sdk/
|
|
@@ -124,6 +124,41 @@ samples = pd.DataFrame({"time": [t0, t0 + 1_000_000_000], "value": [1.0, 2.0]})
|
|
|
124
124
|
dataset.add_signal("car.custom", samples)
|
|
125
125
|
```
|
|
126
126
|
|
|
127
|
+
#### Processing scripts
|
|
128
|
+
|
|
129
|
+
Write a `process(dataset)` function, store it, and try it on any imported dataset. This runs on the server and writes to that dataset.
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
source = """
|
|
133
|
+
from marple.db import Dataset
|
|
134
|
+
|
|
135
|
+
def process(dataset: Dataset) -> None:
|
|
136
|
+
speed = dataset.get_signal("car.speed").get_data()
|
|
137
|
+
dataset.add_signal("car.speed_kmh", speed * 3.6, metadata={"unit": "km/h"})
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
script = db.create_script("speed_kmh", source)
|
|
141
|
+
dataset = stream.get_dataset(path="lap.csv")
|
|
142
|
+
# or: dataset = stream.push_file("lap.csv").wait_for_import()
|
|
143
|
+
dataset.run(script)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Pass a `.py` path or `pathlib.Path` to `create_script` / `script.update` instead of source text. Iterate with `script.update(script=...)` then `dataset.run(script)` again (or `dataset.run(script, source=...)` to save and run in one step). New signals written by the script may need `signal.wait_until_available(...)` before you read them.
|
|
147
|
+
|
|
148
|
+
When the script looks right, attach it to the stream with `stream.update(scripts=...)`. That **replaces** the pipeline (pass `[]` to detach all). New uploads then run those scripts after ingest.
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
stream = stream.update(scripts=[script.id])
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
For files already imported, rerun aliasing and the stream's script pipeline:
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
dataset = dataset.rerun_processing().wait_for_import()
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Use `script.update(...)` to change source or metadata, and `dataset.get_debug_messages()` for the latest ingestion's debug log (not the sandbox job log from `dataset.run`).
|
|
161
|
+
|
|
127
162
|
#### Upload large files
|
|
128
163
|
|
|
129
164
|
`stream.push_file(...)` starts an ingestion and lets the Marple DB API choose the best upload mode. Depending on the deployment and file size, the SDK can upload through the API server, upload directly to Azure Blob Storage, use a single presigned URL, or split the file into multipart uploads.
|
|
@@ -220,6 +255,12 @@ if len(datasets) > 0:
|
|
|
220
255
|
- **Get a resampled df of multiple signals**: `dataset.get_data(signals=[...], resample_rule="1s")`
|
|
221
256
|
- **Delete a stream**: `stream.delete()` or `db.delete_stream(stream_key)`
|
|
222
257
|
- **Delete a dataset**: `dataset.delete()` or `db.delete_dataset(dataset_id, dataset_path)`
|
|
258
|
+
- **Delete signals**: `signal.delete()`, `dataset.delete_signal(signal_id)` / `dataset.delete_signals(signal_ids)`, or `db.delete_signals(dataset_id, dataset_path, signal_ids)`
|
|
259
|
+
- **Run a processing script**: `dataset.run(script)` or `db.run_script(dataset_id, script)`
|
|
260
|
+
- **Create a processing script**: `db.create_script(name, script)`
|
|
261
|
+
- **Set script pipeline** (replaces the full list): `stream.update(scripts=[script.id])`
|
|
262
|
+
- **Rerun aliasing + scripts**: `dataset.rerun_processing().wait_for_import()` or `stream.rerun_processing([dataset.id])`
|
|
263
|
+
- **Read ingest debug logs**: `dataset.get_debug_messages()`
|
|
223
264
|
|
|
224
265
|
For live streams:
|
|
225
266
|
|
|
@@ -89,6 +89,41 @@ samples = pd.DataFrame({"time": [t0, t0 + 1_000_000_000], "value": [1.0, 2.0]})
|
|
|
89
89
|
dataset.add_signal("car.custom", samples)
|
|
90
90
|
```
|
|
91
91
|
|
|
92
|
+
#### Processing scripts
|
|
93
|
+
|
|
94
|
+
Write a `process(dataset)` function, store it, and try it on any imported dataset. This runs on the server and writes to that dataset.
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
source = """
|
|
98
|
+
from marple.db import Dataset
|
|
99
|
+
|
|
100
|
+
def process(dataset: Dataset) -> None:
|
|
101
|
+
speed = dataset.get_signal("car.speed").get_data()
|
|
102
|
+
dataset.add_signal("car.speed_kmh", speed * 3.6, metadata={"unit": "km/h"})
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
script = db.create_script("speed_kmh", source)
|
|
106
|
+
dataset = stream.get_dataset(path="lap.csv")
|
|
107
|
+
# or: dataset = stream.push_file("lap.csv").wait_for_import()
|
|
108
|
+
dataset.run(script)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Pass a `.py` path or `pathlib.Path` to `create_script` / `script.update` instead of source text. Iterate with `script.update(script=...)` then `dataset.run(script)` again (or `dataset.run(script, source=...)` to save and run in one step). New signals written by the script may need `signal.wait_until_available(...)` before you read them.
|
|
112
|
+
|
|
113
|
+
When the script looks right, attach it to the stream with `stream.update(scripts=...)`. That **replaces** the pipeline (pass `[]` to detach all). New uploads then run those scripts after ingest.
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
stream = stream.update(scripts=[script.id])
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
For files already imported, rerun aliasing and the stream's script pipeline:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
dataset = dataset.rerun_processing().wait_for_import()
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Use `script.update(...)` to change source or metadata, and `dataset.get_debug_messages()` for the latest ingestion's debug log (not the sandbox job log from `dataset.run`).
|
|
126
|
+
|
|
92
127
|
#### Upload large files
|
|
93
128
|
|
|
94
129
|
`stream.push_file(...)` starts an ingestion and lets the Marple DB API choose the best upload mode. Depending on the deployment and file size, the SDK can upload through the API server, upload directly to Azure Blob Storage, use a single presigned URL, or split the file into multipart uploads.
|
|
@@ -185,6 +220,12 @@ if len(datasets) > 0:
|
|
|
185
220
|
- **Get a resampled df of multiple signals**: `dataset.get_data(signals=[...], resample_rule="1s")`
|
|
186
221
|
- **Delete a stream**: `stream.delete()` or `db.delete_stream(stream_key)`
|
|
187
222
|
- **Delete a dataset**: `dataset.delete()` or `db.delete_dataset(dataset_id, dataset_path)`
|
|
223
|
+
- **Delete signals**: `signal.delete()`, `dataset.delete_signal(signal_id)` / `dataset.delete_signals(signal_ids)`, or `db.delete_signals(dataset_id, dataset_path, signal_ids)`
|
|
224
|
+
- **Run a processing script**: `dataset.run(script)` or `db.run_script(dataset_id, script)`
|
|
225
|
+
- **Create a processing script**: `db.create_script(name, script)`
|
|
226
|
+
- **Set script pipeline** (replaces the full list): `stream.update(scripts=[script.id])`
|
|
227
|
+
- **Rerun aliasing + scripts**: `dataset.rerun_processing().wait_for_import()` or `stream.rerun_processing([dataset.id])`
|
|
228
|
+
- **Read ingest debug logs**: `dataset.get_debug_messages()`
|
|
188
229
|
|
|
189
230
|
For live streams:
|
|
190
231
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Releasing `marpledata`
|
|
2
|
+
|
|
3
|
+
Tokens: 1Password (`username: __token__`). Run from `python/`.
|
|
4
|
+
|
|
5
|
+
- **Bump version**
|
|
6
|
+
- `uv version --bump minor` / `uv version x.y.z`
|
|
7
|
+
- Manual update `__init__.py`, `CHANGELOG.md`
|
|
8
|
+
- **TestPyPI** — `rm -rf dist && uv build && uv publish --index testpypi`
|
|
9
|
+
Smoke: `pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple marpledata==<version>`
|
|
10
|
+
- **PyPI** — `rm -rf dist && uv build && uv publish`
|
|
11
|
+
- **Docs** — run the GitLab `pages` job (Sphinx → https://marpledata.gitlab.io/marple-sdk/)
|
|
12
|
+
|
|
13
|
+
No GitHub tag/release; `CHANGELOG.md` is the release history.
|
|
@@ -14,6 +14,10 @@ See :doc:`../tutorials` for upload examples.
|
|
|
14
14
|
marple.db.DataStream
|
|
15
15
|
marple.db.Dataset
|
|
16
16
|
marple.db.DatasetList
|
|
17
|
+
marple.db.SandboxJob
|
|
18
|
+
marple.db.SandboxJobStatus
|
|
19
|
+
marple.db.Script
|
|
20
|
+
marple.db.ScriptVersion
|
|
17
21
|
marple.db.Signal
|
|
18
22
|
marple.db.SignalUpload
|
|
19
23
|
marple.db.SignalsAlreadyExistError
|
|
@@ -16,17 +16,21 @@
|
|
|
16
16
|
~DB.add_dataset
|
|
17
17
|
~DB.check_connection
|
|
18
18
|
~DB.connect_trino
|
|
19
|
+
~DB.create_script
|
|
19
20
|
~DB.create_stream
|
|
20
21
|
~DB.dataset_append
|
|
21
22
|
~DB.dataset_cool
|
|
22
23
|
~DB.delete
|
|
23
24
|
~DB.delete_dataset
|
|
25
|
+
~DB.delete_script
|
|
24
26
|
~DB.delete_stream
|
|
25
27
|
~DB.download_original
|
|
26
28
|
~DB.download_signal
|
|
27
29
|
~DB.get
|
|
28
30
|
~DB.get_dataset
|
|
29
31
|
~DB.get_datasets
|
|
32
|
+
~DB.get_script
|
|
33
|
+
~DB.get_scripts
|
|
30
34
|
~DB.get_signal
|
|
31
35
|
~DB.get_signals
|
|
32
36
|
~DB.get_status
|
|
@@ -36,7 +40,10 @@
|
|
|
36
40
|
~DB.post
|
|
37
41
|
~DB.push_file
|
|
38
42
|
~DB.query
|
|
43
|
+
~DB.rerun_processing
|
|
44
|
+
~DB.run_script
|
|
39
45
|
~DB.update_metadata
|
|
46
|
+
~DB.update_stream
|
|
40
47
|
~DB.upsert_signals
|
|
41
48
|
|
|
42
49
|
|
|
@@ -15,9 +15,13 @@
|
|
|
15
15
|
|
|
16
16
|
~DataStream.add_dataset
|
|
17
17
|
~DataStream.delete
|
|
18
|
+
~DataStream.fetch
|
|
18
19
|
~DataStream.get_dataset
|
|
19
20
|
~DataStream.get_datasets
|
|
20
21
|
~DataStream.push_file
|
|
22
|
+
~DataStream.refresh
|
|
23
|
+
~DataStream.rerun_processing
|
|
24
|
+
~DataStream.update
|
|
21
25
|
|
|
22
26
|
|
|
23
27
|
|
|
@@ -46,5 +50,6 @@
|
|
|
46
50
|
~DataStream.plugin
|
|
47
51
|
~DataStream.plugin_args
|
|
48
52
|
~DataStream.signal_reduction
|
|
53
|
+
~DataStream.scripts
|
|
49
54
|
|
|
50
55
|
|
|
@@ -21,8 +21,12 @@
|
|
|
21
21
|
~Dataset.download
|
|
22
22
|
~Dataset.fetch
|
|
23
23
|
~Dataset.get_data
|
|
24
|
+
~Dataset.get_debug_messages
|
|
24
25
|
~Dataset.get_signal
|
|
25
26
|
~Dataset.get_signals
|
|
27
|
+
~Dataset.reingest
|
|
28
|
+
~Dataset.rerun_processing
|
|
29
|
+
~Dataset.run
|
|
26
30
|
~Dataset.update_metadata
|
|
27
31
|
~Dataset.upsert_signals
|
|
28
32
|
~Dataset.wait_for_import
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
marple.db.SandboxJob
|
|
2
|
+
====================
|
|
3
|
+
|
|
4
|
+
.. currentmodule:: marple.db
|
|
5
|
+
|
|
6
|
+
.. autoclass:: SandboxJob
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
.. automethod:: __init__
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
.. rubric:: Methods
|
|
14
|
+
|
|
15
|
+
.. autosummary::
|
|
16
|
+
|
|
17
|
+
~SandboxJob.fetch
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
.. rubric:: Attributes
|
|
24
|
+
|
|
25
|
+
.. autosummary::
|
|
26
|
+
|
|
27
|
+
~SandboxJob.id
|
|
28
|
+
~SandboxJob.dataset_id
|
|
29
|
+
~SandboxJob.stream_id
|
|
30
|
+
~SandboxJob.script_id
|
|
31
|
+
~SandboxJob.script_version
|
|
32
|
+
~SandboxJob.script_index
|
|
33
|
+
~SandboxJob.ingestion_id
|
|
34
|
+
~SandboxJob.status
|
|
35
|
+
~SandboxJob.batch_job_id
|
|
36
|
+
~SandboxJob.token_id
|
|
37
|
+
~SandboxJob.log
|
|
38
|
+
~SandboxJob.created_by
|
|
39
|
+
~SandboxJob.created_at
|
|
40
|
+
~SandboxJob.started_at
|
|
41
|
+
~SandboxJob.finished_at
|
|
42
|
+
|
|
43
|
+
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
marple.db.SandboxJobStatus
|
|
2
|
+
==========================
|
|
3
|
+
|
|
4
|
+
.. currentmodule:: marple.db
|
|
5
|
+
|
|
6
|
+
.. autoclass:: SandboxJobStatus
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
.. rubric:: Methods
|
|
10
|
+
|
|
11
|
+
.. autosummary::
|
|
12
|
+
|
|
13
|
+
~SandboxJobStatus.is_terminal
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
.. rubric:: Members
|
|
20
|
+
|
|
21
|
+
.. autosummary::
|
|
22
|
+
|
|
23
|
+
~SandboxJobStatus.QUEUED
|
|
24
|
+
~SandboxJobStatus.RUNNING
|
|
25
|
+
~SandboxJobStatus.SUCCEEDED
|
|
26
|
+
~SandboxJobStatus.FAILED
|
|
27
|
+
|
|
28
|
+
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
marple.db.Script
|
|
2
|
+
================
|
|
3
|
+
|
|
4
|
+
.. currentmodule:: marple.db
|
|
5
|
+
|
|
6
|
+
.. autoclass:: Script
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
.. automethod:: __init__
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
.. rubric:: Methods
|
|
13
|
+
|
|
14
|
+
.. autosummary::
|
|
15
|
+
|
|
16
|
+
~Script.delete
|
|
17
|
+
~Script.duplicate
|
|
18
|
+
~Script.fetch
|
|
19
|
+
~Script.refresh
|
|
20
|
+
~Script.update
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
.. rubric:: Attributes
|
|
27
|
+
|
|
28
|
+
.. autosummary::
|
|
29
|
+
|
|
30
|
+
~Script.id
|
|
31
|
+
~Script.name
|
|
32
|
+
~Script.description
|
|
33
|
+
~Script.created_at
|
|
34
|
+
~Script.created_by
|
|
35
|
+
~Script.updated_at
|
|
36
|
+
~Script.updated_by
|
|
37
|
+
~Script.streams
|
|
38
|
+
~Script.versions
|
|
39
|
+
~Script.source
|
|
40
|
+
|
|
41
|
+
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
marple.db.ScriptVersion
|
|
2
|
+
=======================
|
|
3
|
+
|
|
4
|
+
.. currentmodule:: marple.db
|
|
5
|
+
|
|
6
|
+
.. autoclass:: ScriptVersion
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
.. automethod:: __init__
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
.. rubric:: Attributes
|
|
13
|
+
|
|
14
|
+
.. autosummary::
|
|
15
|
+
|
|
16
|
+
~ScriptVersion.id
|
|
17
|
+
~ScriptVersion.script
|
|
18
|
+
~ScriptVersion.updated_at
|
|
19
|
+
~ScriptVersion.updated_by
|
|
20
|
+
|
|
21
|
+
|
|
@@ -50,6 +50,13 @@ longer ``wait_for_import`` timeout and optionally increase upload concurrency:
|
|
|
50
50
|
dataset = stream.push_file("large_export.csv", concurrency=8)
|
|
51
51
|
dataset = dataset.wait_for_import(timeout=180)
|
|
52
52
|
|
|
53
|
+
To replace an existing dataset with the same name, pass ``overwrite=True``:
|
|
54
|
+
|
|
55
|
+
.. code-block:: python
|
|
56
|
+
|
|
57
|
+
dataset = stream.push_file("examples_race.csv", overwrite=True)
|
|
58
|
+
dataset = dataset.wait_for_import(timeout=10)
|
|
59
|
+
|
|
53
60
|
If direct storage uploads are blocked by your network or proxy, force upload
|
|
54
61
|
through the Marple DB API server:
|
|
55
62
|
|
|
@@ -31,8 +31,11 @@ Import a file and wait for import
|
|
|
31
31
|
"examples_race.csv",
|
|
32
32
|
metadata={"source": "testbench"},
|
|
33
33
|
concurrency=8,
|
|
34
|
+
overwrite=False,
|
|
34
35
|
).wait_for_import(timeout=180)
|
|
35
36
|
|
|
37
|
+
Pass ``overwrite=True`` to replace an existing dataset with the same name.
|
|
38
|
+
|
|
36
39
|
Add signals to a dataset
|
|
37
40
|
------------------------
|
|
38
41
|
|
|
@@ -75,6 +78,44 @@ For batches, ``add_signals`` returns signal IDs without waiting:
|
|
|
75
78
|
for signal in dataset.get_signals(signal_ids=ids, refresh=True)
|
|
76
79
|
]
|
|
77
80
|
|
|
81
|
+
Processing scripts
|
|
82
|
+
------------------
|
|
83
|
+
|
|
84
|
+
Write a ``process(dataset)`` function, store it, and try it on any imported dataset.
|
|
85
|
+
This runs on the server and writes to that dataset.
|
|
86
|
+
|
|
87
|
+
.. code-block:: python
|
|
88
|
+
|
|
89
|
+
source = """
|
|
90
|
+
from marple.db import Dataset
|
|
91
|
+
|
|
92
|
+
def process(dataset: Dataset) -> None:
|
|
93
|
+
speed = dataset.get_signal("car.speed").get_data()
|
|
94
|
+
dataset.add_signal("car.speed_kmh", speed * 3.6, metadata={"unit": "km/h"})
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
script = db.create_script("speed_kmh", source)
|
|
98
|
+
dataset = stream.get_dataset(path="lap.csv")
|
|
99
|
+
dataset.run(script)
|
|
100
|
+
|
|
101
|
+
Pass a ``.py`` path instead of source text. Iterate with
|
|
102
|
+
``script.update(script=...)`` then ``dataset.run(script)`` (or
|
|
103
|
+
``dataset.run(script, source=...)`` to save and run in one step).
|
|
104
|
+
|
|
105
|
+
When the script looks right, attach it to the stream with
|
|
106
|
+
``stream.update(scripts=...)``. That **replaces** the pipeline
|
|
107
|
+
(pass ``[]`` to detach all). New uploads then run those scripts after ingest.
|
|
108
|
+
|
|
109
|
+
.. code-block:: python
|
|
110
|
+
|
|
111
|
+
stream = stream.update(scripts=[script.id])
|
|
112
|
+
|
|
113
|
+
For files already imported, rerun aliasing and the stream's script pipeline:
|
|
114
|
+
|
|
115
|
+
.. code-block:: python
|
|
116
|
+
|
|
117
|
+
dataset = dataset.rerun_processing().wait_for_import()
|
|
118
|
+
|
|
78
119
|
Filter datasets and get resampled data
|
|
79
120
|
--------------------------------------
|
|
80
121
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "marpledata"
|
|
3
|
-
version = "3.
|
|
3
|
+
version = "3.6.0.dev2"
|
|
4
4
|
description = "Marple SDK for Python"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Matthias Baert", email = "support@marpledata.com" },
|
|
@@ -72,6 +72,11 @@ docs = [
|
|
|
72
72
|
"sphinx>=7.2.6",
|
|
73
73
|
]
|
|
74
74
|
|
|
75
|
+
[tool.pytest.ini_options]
|
|
76
|
+
markers = [
|
|
77
|
+
"integration: tests that talk to a live Marple DB / Insight instance",
|
|
78
|
+
]
|
|
79
|
+
|
|
75
80
|
[tool.black]
|
|
76
81
|
line-length = 113
|
|
77
82
|
target-version = ["py311"]
|
|
@@ -14,6 +14,7 @@ from marple.db.constants import SAAS_URL
|
|
|
14
14
|
from marple.db.constants import SCHEMA as _SCHEMA
|
|
15
15
|
from marple.db.dataset import Dataset, DatasetList
|
|
16
16
|
from marple.db.datastream import DataStream
|
|
17
|
+
from marple.db.script import SandboxJob, SandboxJobStatus, Script, ScriptVersion
|
|
17
18
|
from marple.db.signal import Signal
|
|
18
19
|
from marple.db.signal_upload import SignalsAlreadyExistError, SignalUpload
|
|
19
20
|
from marple.utils import DBClient, validate_response
|
|
@@ -37,6 +38,10 @@ __all__ = [
|
|
|
37
38
|
"DataStream",
|
|
38
39
|
"Dataset",
|
|
39
40
|
"DatasetList",
|
|
41
|
+
"SandboxJob",
|
|
42
|
+
"SandboxJobStatus",
|
|
43
|
+
"Script",
|
|
44
|
+
"ScriptVersion",
|
|
40
45
|
"Signal",
|
|
41
46
|
"SignalUpload",
|
|
42
47
|
"SignalsAlreadyExistError",
|
|
@@ -237,6 +242,89 @@ class DB:
|
|
|
237
242
|
f"Stream with name or id {stream_key} not found, available streams: {', '.join([s.name for s in self._streams.values()])}"
|
|
238
243
|
)
|
|
239
244
|
|
|
245
|
+
def rerun_processing(self, stream_key: str | int, dataset_ids: Sequence[int] | None = None) -> None:
|
|
246
|
+
"""
|
|
247
|
+
Rerun aliasing and processing scripts for a stream or selected datasets.
|
|
248
|
+
|
|
249
|
+
See :meth:`~marple.db.datastream.DataStream.rerun_processing`.
|
|
250
|
+
"""
|
|
251
|
+
self.get_stream(stream_key).rerun_processing(dataset_ids)
|
|
252
|
+
|
|
253
|
+
def get_scripts(self) -> list[Script]:
|
|
254
|
+
"""List processing scripts in this workspace (without source / version history)."""
|
|
255
|
+
r = self.get("/scripts")
|
|
256
|
+
return [
|
|
257
|
+
Script(client=self.client, **script)
|
|
258
|
+
for script in validate_response(r, "Get scripts failed")["scripts"]
|
|
259
|
+
]
|
|
260
|
+
|
|
261
|
+
def get_script(self, script_id: int) -> Script:
|
|
262
|
+
"""Get a processing script by ID, including recent versions and source."""
|
|
263
|
+
return Script.fetch(self.client, script_id)
|
|
264
|
+
|
|
265
|
+
def run_script(
|
|
266
|
+
self,
|
|
267
|
+
dataset_id: int,
|
|
268
|
+
script: Script | int,
|
|
269
|
+
*,
|
|
270
|
+
source: str | Path | None = None,
|
|
271
|
+
version: int | None = None,
|
|
272
|
+
timeout: float = 180,
|
|
273
|
+
) -> Dataset:
|
|
274
|
+
"""Run a stored processing script on a dataset.
|
|
275
|
+
|
|
276
|
+
See :meth:`~marple.db.dataset.Dataset.run`.
|
|
277
|
+
|
|
278
|
+
Args:
|
|
279
|
+
dataset_id: The ID of the dataset to run the script on.
|
|
280
|
+
script: A stored script or its ID.
|
|
281
|
+
source: Optional source text or ``.py`` path to save before running.
|
|
282
|
+
version: Script version ID to run. Defaults to the latest version.
|
|
283
|
+
timeout: Seconds to wait for the sandbox job to finish.
|
|
284
|
+
|
|
285
|
+
Warning:
|
|
286
|
+
This is not a dry-run. The script will be executed and the dataset will be modified.
|
|
287
|
+
"""
|
|
288
|
+
return self.get_dataset(dataset_id).run(script, source=source, version=version, timeout=timeout)
|
|
289
|
+
|
|
290
|
+
def create_script(
|
|
291
|
+
self,
|
|
292
|
+
name: str,
|
|
293
|
+
script: str | Path,
|
|
294
|
+
*,
|
|
295
|
+
description: str | None = None,
|
|
296
|
+
) -> Script:
|
|
297
|
+
"""
|
|
298
|
+
Create a processing script.
|
|
299
|
+
|
|
300
|
+
Attach it to a stream with :meth:`~marple.db.datastream.DataStream.update`
|
|
301
|
+
(``scripts=`` replaces the full pipeline).
|
|
302
|
+
|
|
303
|
+
Args:
|
|
304
|
+
name: The name of the script.
|
|
305
|
+
script: Source text or a path to a file that must define ``process(dataset)``.
|
|
306
|
+
description: The description of the script.
|
|
307
|
+
"""
|
|
308
|
+
r = self.post(
|
|
309
|
+
"/script",
|
|
310
|
+
json={
|
|
311
|
+
"name": name,
|
|
312
|
+
"description": description,
|
|
313
|
+
"script": Script.resolve_source(script),
|
|
314
|
+
},
|
|
315
|
+
)
|
|
316
|
+
return Script(client=self.client, **validate_response(r, "Create script failed"))
|
|
317
|
+
|
|
318
|
+
def delete_script(self, script_id: int) -> None:
|
|
319
|
+
"""
|
|
320
|
+
Delete a processing script.
|
|
321
|
+
|
|
322
|
+
Warning:
|
|
323
|
+
This cannot be undone. The script is removed from every stream pipeline.
|
|
324
|
+
"""
|
|
325
|
+
r = self.delete(f"/script/{script_id}")
|
|
326
|
+
validate_response(r, "Delete script failed")
|
|
327
|
+
|
|
240
328
|
def _refresh_stream_cache(self, r: Response | None = None) -> None:
|
|
241
329
|
if r is None:
|
|
242
330
|
r = self.get("/streams")
|
|
@@ -354,6 +442,7 @@ class DB:
|
|
|
354
442
|
metadata: dict | None = None,
|
|
355
443
|
file_name: str | None = None,
|
|
356
444
|
overwrite: bool = False,
|
|
445
|
+
plugin_args: str | None = None,
|
|
357
446
|
) -> int:
|
|
358
447
|
"""
|
|
359
448
|
Push a file to a datastream.
|
|
@@ -362,12 +451,13 @@ class DB:
|
|
|
362
451
|
- `metadata`: (optional) A dictionary of metadata to be associated with the file.
|
|
363
452
|
- `file_name`: (optional) The name of the file to be stored in the stream.
|
|
364
453
|
- `overwrite`: (optional) If true, existing dataset with the same name will be overwritten.
|
|
454
|
+
- `plugin_args`: (optional) Plugin arguments for this ingest.
|
|
365
455
|
|
|
366
456
|
Note:
|
|
367
457
|
This function is deprecated and it is encouraged to use the `push_file` method in the `DataStream` class directly.
|
|
368
458
|
"""
|
|
369
459
|
stream = self.get_stream(stream_key)
|
|
370
|
-
return stream.push_file(file_path, metadata, file_name, overwrite=overwrite).id
|
|
460
|
+
return stream.push_file(file_path, metadata, file_name, overwrite=overwrite, plugin_args=plugin_args).id
|
|
371
461
|
|
|
372
462
|
@deprecated
|
|
373
463
|
def get_status(self, stream_key: str | int, dataset_id: int) -> dict:
|
|
@@ -432,6 +522,20 @@ class DB:
|
|
|
432
522
|
r = self.post(f"/stream/{dataset.datastream_id}/dataset/{dataset.id}/delete")
|
|
433
523
|
validate_response(r, "Delete dataset failed")
|
|
434
524
|
|
|
525
|
+
def delete_signals(
|
|
526
|
+
self,
|
|
527
|
+
dataset_id: int | None,
|
|
528
|
+
dataset_path: str | None,
|
|
529
|
+
signal_ids: Sequence[int],
|
|
530
|
+
) -> None:
|
|
531
|
+
"""
|
|
532
|
+
Delete signals from a dataset by ID or path.
|
|
533
|
+
|
|
534
|
+
Warning:
|
|
535
|
+
This is a destructive operation that cannot be undone.
|
|
536
|
+
"""
|
|
537
|
+
self.get_dataset(dataset_id, dataset_path).delete_signals(signal_ids)
|
|
538
|
+
|
|
435
539
|
@deprecated
|
|
436
540
|
def update_metadata(
|
|
437
541
|
self,
|