marpledata 3.5.0__tar.gz → 3.6.0.dev2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/AGENTS.md +3 -1
  2. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/CHANGELOG.md +22 -0
  3. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/CONTRIBUTING.md +2 -3
  4. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/PKG-INFO +42 -1
  5. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/README.md +41 -0
  6. marpledata-3.6.0.dev2/RELEASING.md +13 -0
  7. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/db.rst +4 -0
  8. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.DB.rst +7 -0
  9. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.DataStream.rst +5 -0
  10. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.Dataset.rst +4 -0
  11. marpledata-3.6.0.dev2/docs/api/marple.db.SandboxJob.rst +43 -0
  12. marpledata-3.6.0.dev2/docs/api/marple.db.SandboxJobStatus.rst +28 -0
  13. marpledata-3.6.0.dev2/docs/api/marple.db.Script.rst +41 -0
  14. marpledata-3.6.0.dev2/docs/api/marple.db.ScriptVersion.rst +21 -0
  15. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/getting-started.rst +7 -0
  16. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/tutorials.rst +41 -0
  17. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/pyproject.toml +6 -1
  18. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/__init__.py +1 -1
  19. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/__init__.py +105 -1
  20. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/dataset.py +166 -21
  21. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/datastream.py +112 -16
  22. marpledata-3.6.0.dev2/src/marple/db/script.py +182 -0
  23. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/signal.py +13 -0
  24. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/signal_upload.py +1 -1
  25. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/utils.py +26 -15
  26. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/conftest.py +6 -10
  27. marpledata-3.6.0.dev2/tests/support.py +60 -0
  28. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_db.py +111 -83
  29. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_insight.py +3 -1
  30. marpledata-3.6.0.dev2/tests/test_scripts.py +266 -0
  31. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_signal_upload.py +148 -108
  32. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_sql.py +3 -0
  33. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/uv.lock +1 -1
  34. marpledata-3.5.0/RELEASING.md +0 -11
  35. marpledata-3.5.0/tests/support.py +0 -24
  36. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.flake8 +0 -0
  37. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.gitignore +0 -0
  38. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.python-version +0 -0
  39. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/.gitignore +0 -0
  40. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/.lock +0 -0
  41. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/CACHEDIR.TAG +0 -0
  42. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/interpreter-v4/c2c2931c8a99aae1/588e1e7c128e2872.msgpack +0 -0
  43. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/.uv-cache/sdists-v9/.gitignore +0 -0
  44. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/LICENSE +0 -0
  45. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/DEPLOYMENT.md +0 -0
  46. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/_static/custom.css +0 -0
  47. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/_static/favicon.png +0 -0
  48. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/_static/logo.png +0 -0
  49. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/insight.rst +0 -0
  50. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.Insight.rst +0 -0
  51. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.DatasetList.rst +0 -0
  52. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.LAKE_ARROW_SCHEMA.rst +0 -0
  53. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.SCHEMA.rst +0 -0
  54. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.Signal.rst +0 -0
  55. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.SignalUpload.rst +0 -0
  56. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api/marple.db.SignalsAlreadyExistError.rst +0 -0
  57. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/api.rst +0 -0
  58. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/conf.py +0 -0
  59. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/docs/index.rst +0 -0
  60. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/example.py +0 -0
  61. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/examples_race.csv +0 -0
  62. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/pytest.xml +0 -0
  63. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/constants.py +0 -0
  64. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/db/sql.py +0 -0
  65. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/insight.py +0 -0
  66. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/src/marple/py.typed +0 -0
  67. {marpledata-3.5.0 → marpledata-3.6.0.dev2}/tests/test_reliability.py +0 -0
@@ -15,6 +15,7 @@ This directory contains the Python SDK package published as `marpledata`.
15
15
  ## Commands
16
16
 
17
17
  - Run tests with output: `uv run pytest -vs`
18
+ - Unit tests only: `uv run pytest -m "not integration"`
18
19
  - Build docs: `uv run --group docs sphinx-build -b html docs docs/_build/html`
19
20
  - Build package: `uv build`
20
21
  - Fix formatting: `uv run isort src tests && uv run black src tests`
@@ -29,5 +30,6 @@ Run commands from `python/` unless a command explicitly says otherwise.
29
30
  - Keep Python SDK tests in `python/tests/`.
30
31
  - Integration tests run against live Marple services and may require
31
32
  `MDB_TOKEN`, `MDB_URL`, `INSIGHT_TOKEN`, and `INSIGHT_URL`. Tests should skip
32
- or fail clearly when required credentials are missing.
33
+ clearly when required credentials are missing. Mark live tests with
34
+ `pytest.mark.integration`.
33
35
  - Release steps: `RELEASING.md`.
@@ -5,6 +5,28 @@ All notable changes to the Python SDK package `marpledata` will be documented in
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [Unreleased]
9
+
10
+ ### Changed
11
+
12
+ - Exact signal names in `get_data` / `get_signal` / `get_signals` resolve via the dataset cache or `GET /datapool/{pool}/signal/{name}/id` instead of downloading the full datapool `signal_map`. Regex patterns still use the map.
13
+ - Optional `plugin_args` on `DataStream.push_file` (and the deprecated `DB.push_file` wrapper).
14
+
15
+ ### Added
16
+
17
+ - Processing Pipeline
18
+ - `DB.create_script` / `get_scripts` / `get_script` / `delete_script`
19
+ - `Script.update` / `duplicate` / `delete`, to manage stored `process(dataset)` scripts.
20
+ - `Dataset.run` / `DB.run_script` to execute a stored processing script against a dataset via a server-side sandbox job.
21
+ - `DataStream.scripts`, `DataStream.update`, and `DataStream.rerun_processing` (`DB.rerun_processing`) to edit a stream and set the script pipeline
22
+ - `Dataset.rerun_processing` and `Dataset.get_debug_messages`.
23
+ - `DB.delete_signals`, `Dataset.delete_signal` / `Dataset.delete_signals`, and `Signal.delete` to remove signals from a dataset.
24
+ - `Dataset.reingest` to reingest a dataset from its original uploaded file, with optional `plugin_args`.
25
+
26
+ ### Fixed
27
+
28
+ - Allow `add_signal` in post-processing
29
+
8
30
  ## [3.5.0] - 2026-08-20
9
31
 
10
32
  ### Added
@@ -22,11 +22,10 @@ uv run mypy --install-types --non-interactive
22
22
 
23
23
  ### Testing
24
24
 
25
- The testing suite runs against a real linked DB & Insight deployment on our SaaS (e.g. Castro Comrades).
26
- Among other things it will create a stream, ingest a dataset, and export that dataset from Insight
27
- Upload tests also exercise server and multipart upload flows, and create/delete temporary streams with a test prefix.
25
+ The testing suite includes local unit tests and integration tests against a live Marple DB / Insight deployment. Integration tests share one imported example dataset per session, use a small CSV for extra ingestions, and mark themselves with `pytest.mark.integration`.
28
26
 
29
27
  ```bash
28
+ uv run pytest -m "not integration" # local unit tests only
30
29
  export MDB_TOKEN=...
31
30
  export INSIGHT_TOKEN=...
32
31
  # Optional, defaults to SaaS URLs:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: marpledata
3
- Version: 3.5.0
3
+ Version: 3.6.0.dev2
4
4
  Summary: Marple SDK for Python
5
5
  Project-URL: Homepage, https://www.marpledata.com/
6
6
  Project-URL: Documentation, https://marpledata.gitlab.io/marple-sdk/
@@ -124,6 +124,41 @@ samples = pd.DataFrame({"time": [t0, t0 + 1_000_000_000], "value": [1.0, 2.0]})
124
124
  dataset.add_signal("car.custom", samples)
125
125
  ```
126
126
 
127
+ #### Processing scripts
128
+
129
+ Write a `process(dataset)` function, store it, and try it on any imported dataset. This runs on the server and writes to that dataset.
130
+
131
+ ```python
132
+ source = """
133
+ from marple.db import Dataset
134
+
135
+ def process(dataset: Dataset) -> None:
136
+ speed = dataset.get_signal("car.speed").get_data()
137
+ dataset.add_signal("car.speed_kmh", speed * 3.6, metadata={"unit": "km/h"})
138
+ """
139
+
140
+ script = db.create_script("speed_kmh", source)
141
+ dataset = stream.get_dataset(path="lap.csv")
142
+ # or: dataset = stream.push_file("lap.csv").wait_for_import()
143
+ dataset.run(script)
144
+ ```
145
+
146
+ Pass a `.py` path or `pathlib.Path` to `create_script` / `script.update` instead of source text. Iterate with `script.update(script=...)` then `dataset.run(script)` again (or `dataset.run(script, source=...)` to save and run in one step). New signals written by the script may need `signal.wait_until_available(...)` before you read them.
147
+
148
+ When the script looks right, attach it to the stream with `stream.update(scripts=...)`. That **replaces** the pipeline (pass `[]` to detach all). New uploads then run those scripts after ingest.
149
+
150
+ ```python
151
+ stream = stream.update(scripts=[script.id])
152
+ ```
153
+
154
+ For files already imported, rerun aliasing and the stream's script pipeline:
155
+
156
+ ```python
157
+ dataset = dataset.rerun_processing().wait_for_import()
158
+ ```
159
+
160
+ Use `script.update(...)` to change source or metadata, and `dataset.get_debug_messages()` for the latest ingestion's debug log (not the sandbox job log from `dataset.run`).
161
+
127
162
  #### Upload large files
128
163
 
129
164
  `stream.push_file(...)` starts an ingestion and lets the Marple DB API choose the best upload mode. Depending on the deployment and file size, the SDK can upload through the API server, upload directly to Azure Blob Storage, use a single presigned URL, or split the file into multipart uploads.
@@ -220,6 +255,12 @@ if len(datasets) > 0:
220
255
  - **Get a resampled df of multiple signals**: `dataset.get_data(signals=[...], resample_rule="1s")`
221
256
  - **Delete a stream**: `stream.delete()` or `db.delete_stream(stream_key)`
222
257
  - **Delete a dataset**: `dataset.delete()` or `db.delete_dataset(dataset_id, dataset_path)`
258
+ - **Delete signals**: `signal.delete()`, `dataset.delete_signal(signal_id)` / `dataset.delete_signals(signal_ids)`, or `db.delete_signals(dataset_id, dataset_path, signal_ids)`
259
+ - **Run a processing script**: `dataset.run(script)` or `db.run_script(dataset_id, script)`
260
+ - **Create a processing script**: `db.create_script(name, script)`
261
+ - **Set script pipeline** (replaces the full list): `stream.update(scripts=[script.id])`
262
+ - **Rerun aliasing + scripts**: `dataset.rerun_processing().wait_for_import()` or `stream.rerun_processing([dataset.id])`
263
+ - **Read ingest debug logs**: `dataset.get_debug_messages()`
223
264
 
224
265
  For live streams:
225
266
 
@@ -89,6 +89,41 @@ samples = pd.DataFrame({"time": [t0, t0 + 1_000_000_000], "value": [1.0, 2.0]})
89
89
  dataset.add_signal("car.custom", samples)
90
90
  ```
91
91
 
92
+ #### Processing scripts
93
+
94
+ Write a `process(dataset)` function, store it, and try it on any imported dataset. This runs on the server and writes to that dataset.
95
+
96
+ ```python
97
+ source = """
98
+ from marple.db import Dataset
99
+
100
+ def process(dataset: Dataset) -> None:
101
+ speed = dataset.get_signal("car.speed").get_data()
102
+ dataset.add_signal("car.speed_kmh", speed * 3.6, metadata={"unit": "km/h"})
103
+ """
104
+
105
+ script = db.create_script("speed_kmh", source)
106
+ dataset = stream.get_dataset(path="lap.csv")
107
+ # or: dataset = stream.push_file("lap.csv").wait_for_import()
108
+ dataset.run(script)
109
+ ```
110
+
111
+ Pass a `.py` path or `pathlib.Path` to `create_script` / `script.update` instead of source text. Iterate with `script.update(script=...)` then `dataset.run(script)` again (or `dataset.run(script, source=...)` to save and run in one step). New signals written by the script may need `signal.wait_until_available(...)` before you read them.
112
+
113
+ When the script looks right, attach it to the stream with `stream.update(scripts=...)`. That **replaces** the pipeline (pass `[]` to detach all). New uploads then run those scripts after ingest.
114
+
115
+ ```python
116
+ stream = stream.update(scripts=[script.id])
117
+ ```
118
+
119
+ For files already imported, rerun aliasing and the stream's script pipeline:
120
+
121
+ ```python
122
+ dataset = dataset.rerun_processing().wait_for_import()
123
+ ```
124
+
125
+ Use `script.update(...)` to change source or metadata, and `dataset.get_debug_messages()` for the latest ingestion's debug log (not the sandbox job log from `dataset.run`).
126
+
92
127
  #### Upload large files
93
128
 
94
129
  `stream.push_file(...)` starts an ingestion and lets the Marple DB API choose the best upload mode. Depending on the deployment and file size, the SDK can upload through the API server, upload directly to Azure Blob Storage, use a single presigned URL, or split the file into multipart uploads.
@@ -185,6 +220,12 @@ if len(datasets) > 0:
185
220
  - **Get a resampled df of multiple signals**: `dataset.get_data(signals=[...], resample_rule="1s")`
186
221
  - **Delete a stream**: `stream.delete()` or `db.delete_stream(stream_key)`
187
222
  - **Delete a dataset**: `dataset.delete()` or `db.delete_dataset(dataset_id, dataset_path)`
223
+ - **Delete signals**: `signal.delete()`, `dataset.delete_signal(signal_id)` / `dataset.delete_signals(signal_ids)`, or `db.delete_signals(dataset_id, dataset_path, signal_ids)`
224
+ - **Run a processing script**: `dataset.run(script)` or `db.run_script(dataset_id, script)`
225
+ - **Create a processing script**: `db.create_script(name, script)`
226
+ - **Set script pipeline** (replaces the full list): `stream.update(scripts=[script.id])`
227
+ - **Rerun aliasing + scripts**: `dataset.rerun_processing().wait_for_import()` or `stream.rerun_processing([dataset.id])`
228
+ - **Read ingest debug logs**: `dataset.get_debug_messages()`
188
229
 
189
230
  For live streams:
190
231
 
@@ -0,0 +1,13 @@
1
+ # Releasing `marpledata`
2
+
3
+ Tokens: 1Password (`username: __token__`). Run from `python/`.
4
+
5
+ - **Bump version**
6
+ - `uv version --bump minor` / `uv version x.y.z`
7
+ - Manual update `__init__.py`, `CHANGELOG.md`
8
+ - **TestPyPI** — `rm -rf dist && uv build && uv publish --index testpypi`
9
+ Smoke: `pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple marpledata==<version>`
10
+ - **PyPI** — `rm -rf dist && uv build && uv publish`
11
+ - **Docs** — run the GitLab `pages` job (Sphinx → https://marpledata.gitlab.io/marple-sdk/)
12
+
13
+ No GitHub tag/release; `CHANGELOG.md` is the release history.
@@ -14,6 +14,10 @@ See :doc:`../tutorials` for upload examples.
14
14
  marple.db.DataStream
15
15
  marple.db.Dataset
16
16
  marple.db.DatasetList
17
+ marple.db.SandboxJob
18
+ marple.db.SandboxJobStatus
19
+ marple.db.Script
20
+ marple.db.ScriptVersion
17
21
  marple.db.Signal
18
22
  marple.db.SignalUpload
19
23
  marple.db.SignalsAlreadyExistError
@@ -16,17 +16,21 @@
16
16
  ~DB.add_dataset
17
17
  ~DB.check_connection
18
18
  ~DB.connect_trino
19
+ ~DB.create_script
19
20
  ~DB.create_stream
20
21
  ~DB.dataset_append
21
22
  ~DB.dataset_cool
22
23
  ~DB.delete
23
24
  ~DB.delete_dataset
25
+ ~DB.delete_script
24
26
  ~DB.delete_stream
25
27
  ~DB.download_original
26
28
  ~DB.download_signal
27
29
  ~DB.get
28
30
  ~DB.get_dataset
29
31
  ~DB.get_datasets
32
+ ~DB.get_script
33
+ ~DB.get_scripts
30
34
  ~DB.get_signal
31
35
  ~DB.get_signals
32
36
  ~DB.get_status
@@ -36,7 +40,10 @@
36
40
  ~DB.post
37
41
  ~DB.push_file
38
42
  ~DB.query
43
+ ~DB.rerun_processing
44
+ ~DB.run_script
39
45
  ~DB.update_metadata
46
+ ~DB.update_stream
40
47
  ~DB.upsert_signals
41
48
 
42
49
 
@@ -15,9 +15,13 @@
15
15
 
16
16
  ~DataStream.add_dataset
17
17
  ~DataStream.delete
18
+ ~DataStream.fetch
18
19
  ~DataStream.get_dataset
19
20
  ~DataStream.get_datasets
20
21
  ~DataStream.push_file
22
+ ~DataStream.refresh
23
+ ~DataStream.rerun_processing
24
+ ~DataStream.update
21
25
 
22
26
 
23
27
 
@@ -46,5 +50,6 @@
46
50
  ~DataStream.plugin
47
51
  ~DataStream.plugin_args
48
52
  ~DataStream.signal_reduction
53
+ ~DataStream.scripts
49
54
 
50
55
 
@@ -21,8 +21,12 @@
21
21
  ~Dataset.download
22
22
  ~Dataset.fetch
23
23
  ~Dataset.get_data
24
+ ~Dataset.get_debug_messages
24
25
  ~Dataset.get_signal
25
26
  ~Dataset.get_signals
27
+ ~Dataset.reingest
28
+ ~Dataset.rerun_processing
29
+ ~Dataset.run
26
30
  ~Dataset.update_metadata
27
31
  ~Dataset.upsert_signals
28
32
  ~Dataset.wait_for_import
@@ -0,0 +1,43 @@
1
+ marple.db.SandboxJob
2
+ ====================
3
+
4
+ .. currentmodule:: marple.db
5
+
6
+ .. autoclass:: SandboxJob
7
+
8
+
9
+ .. automethod:: __init__
10
+
11
+
12
+
13
+ .. rubric:: Methods
14
+
15
+ .. autosummary::
16
+
17
+ ~SandboxJob.fetch
18
+
19
+
20
+
21
+
22
+
23
+ .. rubric:: Attributes
24
+
25
+ .. autosummary::
26
+
27
+ ~SandboxJob.id
28
+ ~SandboxJob.dataset_id
29
+ ~SandboxJob.stream_id
30
+ ~SandboxJob.script_id
31
+ ~SandboxJob.script_version
32
+ ~SandboxJob.script_index
33
+ ~SandboxJob.ingestion_id
34
+ ~SandboxJob.status
35
+ ~SandboxJob.batch_job_id
36
+ ~SandboxJob.token_id
37
+ ~SandboxJob.log
38
+ ~SandboxJob.created_by
39
+ ~SandboxJob.created_at
40
+ ~SandboxJob.started_at
41
+ ~SandboxJob.finished_at
42
+
43
+
@@ -0,0 +1,28 @@
1
+ marple.db.SandboxJobStatus
2
+ ==========================
3
+
4
+ .. currentmodule:: marple.db
5
+
6
+ .. autoclass:: SandboxJobStatus
7
+
8
+
9
+ .. rubric:: Methods
10
+
11
+ .. autosummary::
12
+
13
+ ~SandboxJobStatus.is_terminal
14
+
15
+
16
+
17
+
18
+
19
+ .. rubric:: Members
20
+
21
+ .. autosummary::
22
+
23
+ ~SandboxJobStatus.QUEUED
24
+ ~SandboxJobStatus.RUNNING
25
+ ~SandboxJobStatus.SUCCEEDED
26
+ ~SandboxJobStatus.FAILED
27
+
28
+
@@ -0,0 +1,41 @@
1
+ marple.db.Script
2
+ ================
3
+
4
+ .. currentmodule:: marple.db
5
+
6
+ .. autoclass:: Script
7
+
8
+
9
+ .. automethod:: __init__
10
+
11
+
12
+ .. rubric:: Methods
13
+
14
+ .. autosummary::
15
+
16
+ ~Script.delete
17
+ ~Script.duplicate
18
+ ~Script.fetch
19
+ ~Script.refresh
20
+ ~Script.update
21
+
22
+
23
+
24
+
25
+
26
+ .. rubric:: Attributes
27
+
28
+ .. autosummary::
29
+
30
+ ~Script.id
31
+ ~Script.name
32
+ ~Script.description
33
+ ~Script.created_at
34
+ ~Script.created_by
35
+ ~Script.updated_at
36
+ ~Script.updated_by
37
+ ~Script.streams
38
+ ~Script.versions
39
+ ~Script.source
40
+
41
+
@@ -0,0 +1,21 @@
1
+ marple.db.ScriptVersion
2
+ =======================
3
+
4
+ .. currentmodule:: marple.db
5
+
6
+ .. autoclass:: ScriptVersion
7
+
8
+
9
+ .. automethod:: __init__
10
+
11
+
12
+ .. rubric:: Attributes
13
+
14
+ .. autosummary::
15
+
16
+ ~ScriptVersion.id
17
+ ~ScriptVersion.script
18
+ ~ScriptVersion.updated_at
19
+ ~ScriptVersion.updated_by
20
+
21
+
@@ -50,6 +50,13 @@ longer ``wait_for_import`` timeout and optionally increase upload concurrency:
50
50
  dataset = stream.push_file("large_export.csv", concurrency=8)
51
51
  dataset = dataset.wait_for_import(timeout=180)
52
52
 
53
+ To replace an existing dataset with the same name, pass ``overwrite=True``:
54
+
55
+ .. code-block:: python
56
+
57
+ dataset = stream.push_file("examples_race.csv", overwrite=True)
58
+ dataset = dataset.wait_for_import(timeout=10)
59
+
53
60
  If direct storage uploads are blocked by your network or proxy, force upload
54
61
  through the Marple DB API server:
55
62
 
@@ -31,8 +31,11 @@ Import a file and wait for import
31
31
  "examples_race.csv",
32
32
  metadata={"source": "testbench"},
33
33
  concurrency=8,
34
+ overwrite=False,
34
35
  ).wait_for_import(timeout=180)
35
36
 
37
+ Pass ``overwrite=True`` to replace an existing dataset with the same name.
38
+
36
39
  Add signals to a dataset
37
40
  ------------------------
38
41
 
@@ -75,6 +78,44 @@ For batches, ``add_signals`` returns signal IDs without waiting:
75
78
  for signal in dataset.get_signals(signal_ids=ids, refresh=True)
76
79
  ]
77
80
 
81
+ Processing scripts
82
+ ------------------
83
+
84
+ Write a ``process(dataset)`` function, store it, and try it on any imported dataset.
85
+ This runs on the server and writes to that dataset.
86
+
87
+ .. code-block:: python
88
+
89
+ source = """
90
+ from marple.db import Dataset
91
+
92
+ def process(dataset: Dataset) -> None:
93
+ speed = dataset.get_signal("car.speed").get_data()
94
+ dataset.add_signal("car.speed_kmh", speed * 3.6, metadata={"unit": "km/h"})
95
+ """
96
+
97
+ script = db.create_script("speed_kmh", source)
98
+ dataset = stream.get_dataset(path="lap.csv")
99
+ dataset.run(script)
100
+
101
+ Pass a ``.py`` path instead of source text. Iterate with
102
+ ``script.update(script=...)`` then ``dataset.run(script)`` (or
103
+ ``dataset.run(script, source=...)`` to save and run in one step).
104
+
105
+ When the script looks right, attach it to the stream with
106
+ ``stream.update(scripts=...)``. That **replaces** the pipeline
107
+ (pass ``[]`` to detach all). New uploads then run those scripts after ingest.
108
+
109
+ .. code-block:: python
110
+
111
+ stream = stream.update(scripts=[script.id])
112
+
113
+ For files already imported, rerun aliasing and the stream's script pipeline:
114
+
115
+ .. code-block:: python
116
+
117
+ dataset = dataset.rerun_processing().wait_for_import()
118
+
78
119
  Filter datasets and get resampled data
79
120
  --------------------------------------
80
121
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "marpledata"
3
- version = "3.5.0"
3
+ version = "3.6.0.dev2"
4
4
  description = "Marple SDK for Python"
5
5
  authors = [
6
6
  { name = "Matthias Baert", email = "support@marpledata.com" },
@@ -72,6 +72,11 @@ docs = [
72
72
  "sphinx>=7.2.6",
73
73
  ]
74
74
 
75
+ [tool.pytest.ini_options]
76
+ markers = [
77
+ "integration: tests that talk to a live Marple DB / Insight instance",
78
+ ]
79
+
75
80
  [tool.black]
76
81
  line-length = 113
77
82
  target-version = ["py311"]
@@ -3,4 +3,4 @@ from .db import DB
3
3
  from .insight import Insight
4
4
 
5
5
  __all__ = ["DB", "Insight", "db"]
6
- __version__ = "3.5.0"
6
+ __version__ = "3.6.0.dev2"
@@ -14,6 +14,7 @@ from marple.db.constants import SAAS_URL
14
14
  from marple.db.constants import SCHEMA as _SCHEMA
15
15
  from marple.db.dataset import Dataset, DatasetList
16
16
  from marple.db.datastream import DataStream
17
+ from marple.db.script import SandboxJob, SandboxJobStatus, Script, ScriptVersion
17
18
  from marple.db.signal import Signal
18
19
  from marple.db.signal_upload import SignalsAlreadyExistError, SignalUpload
19
20
  from marple.utils import DBClient, validate_response
@@ -37,6 +38,10 @@ __all__ = [
37
38
  "DataStream",
38
39
  "Dataset",
39
40
  "DatasetList",
41
+ "SandboxJob",
42
+ "SandboxJobStatus",
43
+ "Script",
44
+ "ScriptVersion",
40
45
  "Signal",
41
46
  "SignalUpload",
42
47
  "SignalsAlreadyExistError",
@@ -237,6 +242,89 @@ class DB:
237
242
  f"Stream with name or id {stream_key} not found, available streams: {', '.join([s.name for s in self._streams.values()])}"
238
243
  )
239
244
 
245
+ def rerun_processing(self, stream_key: str | int, dataset_ids: Sequence[int] | None = None) -> None:
246
+ """
247
+ Rerun aliasing and processing scripts for a stream or selected datasets.
248
+
249
+ See :meth:`~marple.db.datastream.DataStream.rerun_processing`.
250
+ """
251
+ self.get_stream(stream_key).rerun_processing(dataset_ids)
252
+
253
+ def get_scripts(self) -> list[Script]:
254
+ """List processing scripts in this workspace (without source / version history)."""
255
+ r = self.get("/scripts")
256
+ return [
257
+ Script(client=self.client, **script)
258
+ for script in validate_response(r, "Get scripts failed")["scripts"]
259
+ ]
260
+
261
+ def get_script(self, script_id: int) -> Script:
262
+ """Get a processing script by ID, including recent versions and source."""
263
+ return Script.fetch(self.client, script_id)
264
+
265
+ def run_script(
266
+ self,
267
+ dataset_id: int,
268
+ script: Script | int,
269
+ *,
270
+ source: str | Path | None = None,
271
+ version: int | None = None,
272
+ timeout: float = 180,
273
+ ) -> Dataset:
274
+ """Run a stored processing script on a dataset.
275
+
276
+ See :meth:`~marple.db.dataset.Dataset.run`.
277
+
278
+ Args:
279
+ dataset_id: The ID of the dataset to run the script on.
280
+ script: A stored script or its ID.
281
+ source: Optional source text or ``.py`` path to save before running.
282
+ version: Script version ID to run. Defaults to the latest version.
283
+ timeout: Seconds to wait for the sandbox job to finish.
284
+
285
+ Warning:
286
+ This is not a dry-run. The script will be executed and the dataset will be modified.
287
+ """
288
+ return self.get_dataset(dataset_id).run(script, source=source, version=version, timeout=timeout)
289
+
290
+ def create_script(
291
+ self,
292
+ name: str,
293
+ script: str | Path,
294
+ *,
295
+ description: str | None = None,
296
+ ) -> Script:
297
+ """
298
+ Create a processing script.
299
+
300
+ Attach it to a stream with :meth:`~marple.db.datastream.DataStream.update`
301
+ (``scripts=`` replaces the full pipeline).
302
+
303
+ Args:
304
+ name: The name of the script.
305
+ script: Source text or a path to a file that must define ``process(dataset)``.
306
+ description: The description of the script.
307
+ """
308
+ r = self.post(
309
+ "/script",
310
+ json={
311
+ "name": name,
312
+ "description": description,
313
+ "script": Script.resolve_source(script),
314
+ },
315
+ )
316
+ return Script(client=self.client, **validate_response(r, "Create script failed"))
317
+
318
+ def delete_script(self, script_id: int) -> None:
319
+ """
320
+ Delete a processing script.
321
+
322
+ Warning:
323
+ This cannot be undone. The script is removed from every stream pipeline.
324
+ """
325
+ r = self.delete(f"/script/{script_id}")
326
+ validate_response(r, "Delete script failed")
327
+
240
328
  def _refresh_stream_cache(self, r: Response | None = None) -> None:
241
329
  if r is None:
242
330
  r = self.get("/streams")
@@ -354,6 +442,7 @@ class DB:
354
442
  metadata: dict | None = None,
355
443
  file_name: str | None = None,
356
444
  overwrite: bool = False,
445
+ plugin_args: str | None = None,
357
446
  ) -> int:
358
447
  """
359
448
  Push a file to a datastream.
@@ -362,12 +451,13 @@ class DB:
362
451
  - `metadata`: (optional) A dictionary of metadata to be associated with the file.
363
452
  - `file_name`: (optional) The name of the file to be stored in the stream.
364
453
  - `overwrite`: (optional) If true, existing dataset with the same name will be overwritten.
454
+ - `plugin_args`: (optional) Plugin arguments for this ingest.
365
455
 
366
456
  Note:
367
457
  This function is deprecated and it is encouraged to use the `push_file` method in the `DataStream` class directly.
368
458
  """
369
459
  stream = self.get_stream(stream_key)
370
- return stream.push_file(file_path, metadata, file_name, overwrite=overwrite).id
460
+ return stream.push_file(file_path, metadata, file_name, overwrite=overwrite, plugin_args=plugin_args).id
371
461
 
372
462
  @deprecated
373
463
  def get_status(self, stream_key: str | int, dataset_id: int) -> dict:
@@ -432,6 +522,20 @@ class DB:
432
522
  r = self.post(f"/stream/{dataset.datastream_id}/dataset/{dataset.id}/delete")
433
523
  validate_response(r, "Delete dataset failed")
434
524
 
525
+ def delete_signals(
526
+ self,
527
+ dataset_id: int | None,
528
+ dataset_path: str | None,
529
+ signal_ids: Sequence[int],
530
+ ) -> None:
531
+ """
532
+ Delete signals from a dataset by ID or path.
533
+
534
+ Warning:
535
+ This is a destructive operation that cannot be undone.
536
+ """
537
+ self.get_dataset(dataset_id, dataset_path).delete_signals(signal_ids)
538
+
435
539
  @deprecated
436
540
  def update_metadata(
437
541
  self,