marpledata 3.0.0.dev1__tar.gz → 3.1.0__tar.gz

{marpledata-3.0.0.dev1 → marpledata-3.1.0}/.gitignore

@@ -1,3 +1,10 @@
+lol.py
+.mdb_cache/
+matlab/bin/
+
+python/docs/_build/
+python/docs/api/marple.*.rst
+
 # Default ignored files
 /shelf/
 /workspace.xml
@@ -161,4 +168,5 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-.idea/
+.idea/
+*.env

{marpledata-3.0.0.dev1 → marpledata-3.1.0}/CONTRIBUTING.md

@@ -11,6 +11,17 @@ export INSIGHT_TOKEN=...
 uv run pytest -vs
 ```
 
+### Documentation
+
+Build the Sphinx docs locally:
+
+```bash
+uv sync --group docs
+uv run sphinx-build -b html docs docs/_build/html
+```
+
+Open `docs/_build/html/index.html` in your browser to view the site.
+
 ### Local build
 
 - `uv build`
@@ -18,7 +29,7 @@ uv run pytest -vs
 - `uv run python`
   - `import marple`
   - `marple.__version__`
-  - `from marple import Marple, Insight, DB`
+  - `from marple import Insight, DB`
 
 ### Publishing (Test Pypi)
 
@@ -59,3 +70,4 @@ To ensure pip correctly updates our package, correct versioning is important. Us
 - `uv config pypi-token.pypi pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXX`
 - `uv build`
 - `uv publish`
+- Run the GitLab pipeline `pages` to build & release the docs

{marpledata-3.0.0.dev1 → marpledata-3.1.0}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: marpledata
-Version: 3.0.0.dev1
+Version: 3.1.0
 Summary: Marple SDK for Python
 Project-URL: Homepage, https://www.marpledata.com/
-Project-URL: Documentation, https://docs.marpledata.com/docs
+Project-URL: Documentation, https://marpledata.gitlab.io/marple-sdk/
 Project-URL: Repository, https://gitlab.com/marpledata/marple-sdk/-/tree/main/python
 Project-URL: Issues, https://github.com/marpledata/marple-sdk/issues
 Author-email: Matthias Baert <support@marpledata.com>, Jan Scheers <jan@marpledata.com>
@@ -28,6 +28,7 @@ Requires-Dist: pandas>=2.3.3
 Requires-Dist: pyarrow>=22.0.0
 Requires-Dist: pydantic>=2.12.5
 Requires-Dist: requests>=2.32.5
+Requires-Dist: tabulate>=0.9.0
 Description-Content-Type: text/markdown
 
 # Marple SDK
@@ -58,7 +59,8 @@ To get started:
 
 If you are using a VPC or self-hosted version, pass a custom `api_url` to `DB(...)` (it should end in `/api/v1`).
 
-### Example: import a file and poll ingest status
+### Examples
+#### Import a file and wait for it to import
 
 This is the typical flow for importing a new file into Marple DB:
 
@@ -75,23 +77,77 @@ db = DB(API_TOKEN, API_URL)
 
 db.check_connection()
 
-dataset_id = db.push_file(STREAM, "tests/examples_race.csv", metadata={"driver": "Mbaerto"})
+stream = db.get_stream(STREAM)
+dataset = stream.push_file("examples_race.csv", metadata={"driver": "Mbaerto"})
+# Wait at most 10s for the dataset to completely import and get the new state of the dataset
+dataset = dataset.wait_for_import(timeout=10)
+```
+
+#### Filter datasets and get resampled data
+```python
+# See previous example for setup
+import re
+from marple.db import Dataset
+
+datasets = stream.get_datasets()  # Get all datasets in a specific Data Stream
+# OR
+# datasets = db.get_datasets()  # Get all datasets in the datapool
+
+datasets = (
+    datasets
+    .where_metadata({"car_id": [1, 2], "track": "track_1"})
+    .wait_for_import()
+    .where_imported()
+    .where_signal("car.speed", "max", greater_than=75)
+    .where_signal("car.engine.temp", "mean", greater_than=30)
+    .where_dataset("n_datapoints", greater_than=100000)
+)
+
+def custom_filter_function(dataset: Dataset) -> bool:
+    return (
+        dataset.metadata.get("weather") == "sunny"
+        or dataset.get_signal("car.engine.NGear").stats.get("avg", 0) ** 2 > 16
+    )
+
+# Pass any function to filter the datasets on more complex conditions
+datasets = datasets.where(custom_filter_function)
+
+# Create an overview of the datasets as a pandas.DataFrame to save it to a CSV.
+datasets.to_pandas().to_csv("all_datasets.csv")
+
+# Get a dataframe per dataset of the matching signals which is resampled at a period of 0.17s.
+# The regex patterns will match with car.wheel.rear.left.speed, car.wheel.rear.front.speed, ...
+for dataset, data in datasets.get_data(
+    signals=[
+        "car.speed",
+        "car.engine.temp",
+        re.compile("car.wheel.*.speed"),
+        re.compile("car.wheel.*.trq"),
+    ],
+    resample_rule="0.17s",
+):
+    machine_learning_model.train(data)
+```
+
+#### Delete a dataset that failed to import
+```python
+datasets = stream.get_datasets()
+datasets = datasets.where_dataset("import_status", equals="FAILED")
 
-while True:
-    status = db.get_status(STREAM, dataset_id)
-    if status.get("import_status") in {"FINISHED", "FAILED"}:
-        break
-    time.sleep(1)
+# datasets is of type DatasetList which is a subclass of list so you can do all normal list operations on it.
+if len(datasets) > 0:
+    datasets[0].delete()
 ```
 
 ### Common operations
 
 - **List streams**: `db.get_streams()`
-- **List datasets in a stream**: `db.get_datasets(stream_key)`
-- **Upload a file to a file-stream**: `db.push_file(stream_key, file_path, metadata={...})`
-- **Poll ingest status**: `db.get_status(stream_key, dataset_id)`
-- **Download original uploaded file**: `db.download_original(stream_key, dataset_id, destination_folder=".")`
-- **Download parquet for a signal**: `db.download_signal(stream_key, dataset_id, signal_id, destination_folder=".")`
+- **List datasets in a stream**: `stream.get_datasets()`
+- **Upload a file to a file-stream**: `stream.push_file(file_path, metadata={...})`
+- **Wait for a dataset to import**: `dataset.wait_for_import(timeout=60)`
+- **Download original uploaded file**: `dataset.download(destination_folder=".")`
+- **Download parquet for a signal**: `dataset.get_signal(signal_name).download(destination_folder=".")`
+- **Get a resampled df of multiple signals**: `dataset.get_data(signals=[...], resample_rule="1s")`
 
 For live/realtime streams (creating and appending data):
 
@@ -105,9 +161,14 @@ For advanced use cases, you can call API endpoints directly:
 
 ```python
 db.get("/health")
-db.post("/query", json={"query": "select 1"})
 ```
 
+### Notes on DB API changes
+
+- Methods like `DB.push_file`, `DB.download_signal`, and `DB.update_metadata` are deprecated.
+- Prefer stream/dataset methods instead: `stream.push_file`, `dataset.get_signal(...).download()`, and `dataset.update_metadata(...)`.
+- These are still available for compatibility, but the examples above use the current API.
+
 ## Marple Insight
 
 ### Common operations
@@ -130,7 +191,7 @@ STREAM = "Car data"
 insight = Insight(INSIGHT_TOKEN, INSIGHT_URL)
 db = DB(DB_TOKEN, DB_URL)
 
-dataset_id = db.get_datasets(STREAM)[0]["id"]
+dataset_id = db.get_datasets(STREAM)[0].id
 insight_dataset = insight.get_dataset_mdb(dataset_id)
 
 file_path = insight.export_data_mdb(
@@ -141,3 +202,10 @@ file_path = insight.export_data_mdb(
 )
 print("Wrote", file_path)
 ```
+
+### Calling endpoints directly
+For advanced use cases, you can call API endpoints directly:
+```python
+insight.get("/user/info")
+insight.post("sources/search")
+```

marpledata-3.1.0/README.md

@@ -0,0 +1,178 @@
+# Marple SDK
+
+An SDK to interact with [Marple](https://www.marpledata.com) DB & Insight
+
+## Installation and importing
+
+Install the Marple SDK using your package manager:
+
+- `poetry add marpledata`
+- `uv add marpledata`
+- `pip install marpledata`
+
+The SDK currently exposes:
+
+```python
+from marple import DB      # Marple DB
+from marple import Insight # Marple Insight
+```
+
+## Marple DB
+
+To get started:
+
+- Create a **stream** in the Marple DB UI
+- Create an **API token** (in user settings)
+
+If you are using a VPC or self-hosted version, pass a custom `api_url` to `DB(...)` (it should end in `/api/v1`).
+
+### Examples
+#### Import a file and wait for it to import
+
+This is the typical flow for importing a new file into Marple DB:
+
+```python
+import time
+from marple import DB
+
+# Create a stream + API token in the Marple DB web application
+STREAM = "Car data"
+API_TOKEN = "<your api token>"
+API_URL = "https://db.marpledata.com/api/v1"  # optional if using the default SaaS
+
+db = DB(API_TOKEN, API_URL)
+
+db.check_connection()
+
+stream = db.get_stream(STREAM)
+dataset = stream.push_file("examples_race.csv", metadata={"driver": "Mbaerto"})
+# Wait at most 10s for the dataset to completely import and get the new state of the dataset
+dataset = dataset.wait_for_import(timeout=10)
+```
+
+#### Filter datasets and get resampled data
+```python
+# See previous example for setup
+import re
+from marple.db import Dataset
+
+datasets = stream.get_datasets()  # Get all datasets in a specific Data Stream
+# OR
+# datasets = db.get_datasets()  # Get all datasets in the datapool
+
+datasets = (
+    datasets
+    .where_metadata({"car_id": [1, 2], "track": "track_1"})
+    .wait_for_import()
+    .where_imported()
+    .where_signal("car.speed", "max", greater_than=75)
+    .where_signal("car.engine.temp", "mean", greater_than=30)
+    .where_dataset("n_datapoints", greater_than=100000)
+)
+
+def custom_filter_function(dataset: Dataset) -> bool:
+    return (
+        dataset.metadata.get("weather") == "sunny"
+        or dataset.get_signal("car.engine.NGear").stats.get("avg", 0) ** 2 > 16
+    )
+
+# Pass any function to filter the datasets on more complex conditions
+datasets = datasets.where(custom_filter_function)
+
+# Create an overview of the datasets as a pandas.DataFrame to save it to a CSV.
+datasets.to_pandas().to_csv("all_datasets.csv")
+
+# Get a dataframe per dataset of the matching signals which is resampled at a period of 0.17s.
+# The regex patterns will match with car.wheel.rear.left.speed, car.wheel.rear.front.speed, ...
+for dataset, data in datasets.get_data(
+    signals=[
+        "car.speed",
+        "car.engine.temp",
+        re.compile("car.wheel.*.speed"),
+        re.compile("car.wheel.*.trq"),
+    ],
+    resample_rule="0.17s",
+):
+    machine_learning_model.train(data)
+```
+
+#### Delete a dataset that failed to import
+```python
+datasets = stream.get_datasets()
+datasets = datasets.where_dataset("import_status", equals="FAILED")
+
+# datasets is of type DatasetList which is a subclass of list so you can do all normal list operations on it.
+if len(datasets) > 0:
+    datasets[0].delete()
+```
+
+### Common operations
+
+- **List streams**: `db.get_streams()`
+- **List datasets in a stream**: `stream.get_datasets()`
+- **Upload a file to a file-stream**: `stream.push_file(file_path, metadata={...})`
+- **Wait for a dataset to import**: `dataset.wait_for_import(timeout=60)`
+- **Download original uploaded file**: `dataset.download(destination_folder=".")`
+- **Download parquet for a signal**: `dataset.get_signal(signal_name).download(destination_folder=".")`
+- **Get a resampled df of multiple signals**: `dataset.get_data(signals=[...], resample_rule="1s")`
+
+For live/realtime streams (creating and appending data):
+
+- **Create an empty dataset**: `db.add_dataset(stream_key, dataset_name, metadata=None)`
+- **Upsert signal definitions**: `db.upsert_signals(stream_key, dataset_id, signals=[...])`
+- **Append timeseries data**: `db.dataset_append(stream_key, dataset_id, data=df, shape="long"|"wide"|None)`
+
+### Calling endpoints directly
+
+For advanced use cases, you can call API endpoints directly:
+
+```python
+db.get("/health")
+```
+
+### Notes on DB API changes
+
+- Methods like `DB.push_file`, `DB.download_signal`, and `DB.update_metadata` are deprecated.
+- Prefer stream/dataset methods instead: `stream.push_file`, `dataset.get_signal(...).download()`, and `dataset.update_metadata(...)`.
+- These are still available for compatibility, but the examples above use the current API.
+
+## Marple Insight
+
+### Common operations
+
+- **List datasets in the workspace**: `insight.get_datasets()`
+- **Get a Marple DB dataset (by dataset id)**: `insight.get_dataset_mdb(dataset_id)`
+- **List signals in a dataset**: `insight.get_signals(dataset_filter)` / `insight.get_signals_mdb(dataset_id)`
+
+### Example: export a dataset (H5/MAT)
+
+```python
+from marple import DB, Insight
+
+INSIGHT_TOKEN = "<your api token>"
+INSIGHT_URL = "https://insight.marpledata.com/api/v1"  # optional if using the default SaaS
+DB_TOKEN = "<your api token>"
+DB_URL = "https://db.marpledata.com/api/v1"  # optional if using the default SaaS
+STREAM = "Car data"
+
+insight = Insight(INSIGHT_TOKEN, INSIGHT_URL)
+db = DB(DB_TOKEN, DB_URL)
+
+dataset_id = db.get_datasets(STREAM)[0].id
+insight_dataset = insight.get_dataset_mdb(dataset_id)
+
+file_path = insight.export_data_mdb(
+    dataset_id,
+    format="h5",
+    signals=["car.speed"],
+    destination=".",
+)
+print("Wrote", file_path)
+```
+
+### Calling endpoints directly
+For advanced use cases, you can call API endpoints directly:
+```python
+insight.get("/user/info")
+insight.post("sources/search")
+```

marpledata-3.1.0/docs/DEPLOYMENT.md

@@ -0,0 +1,24 @@
+# Documentation Deployment
+
+The Sphinx documentation can be manually built and deployed to GitLab Pages with a single click.
+
+## How to Deploy
+Click **Run pipeline** to trigger it
+The`pages` job will start and automatically build the docs and serve them as a static gitlab page under `https://marpledata.gitlab.io/marple-sdk/`
+
+## Local Development
+
+To build docs locally:
+```bash
+cd python
+uv run sphinx-build -b html docs docs/_build/html
+```
+
+Then open `docs/_build/html/index.html` in your browser.
+
+
+**Sphinx configuration:** `python/docs/conf.py`
+- Uses PyData theme
+- Includes Pydantic filtering to hide internal methods
+- 2-product structure (Insight, DB, Files)
+

marpledata-3.1.0/docs/_static/custom.css

@@ -0,0 +1,4 @@
+:root {
+    /* The primary color used for links and active states */
+    --pst-color-primary: #0253A3;
+}

marpledata-3.1.0/docs/api/db.rst

@@ -0,0 +1,12 @@
+Marple DB
+=========
+
+.. autosummary::
+   :toctree: .
+   :nosignatures:
+
+   marple.db.DB
+   marple.db.DataStream
+   marple.db.Dataset
+   marple.db.DatasetList
+   marple.db.Signal

marpledata-3.1.0/docs/api/insight.rst

@@ -0,0 +1,8 @@
+Marple Insight
+==============
+
+.. autosummary::
+   :toctree: .
+   :nosignatures:
+
+   marple.Insight

marpledata-3.1.0/docs/api/marple.Insight.rst

@@ -0,0 +1,33 @@
+marple.Insight
+==============
+
+.. currentmodule:: marple
+
+.. autoclass:: Insight
+
+   
+   .. automethod:: __init__
+
+   
+   .. rubric:: Methods
+
+   .. autosummary::
+   
+      ~Insight.check_connection
+      ~Insight.delete
+      ~Insight.export_data
+      ~Insight.export_data_mdb
+      ~Insight.get
+      ~Insight.get_dataset
+      ~Insight.get_dataset_mdb
+      ~Insight.get_datasets
+      ~Insight.get_signals
+      ~Insight.get_signals_mdb
+      ~Insight.patch
+      ~Insight.post
+   
+   
+
+   
+   
+   

marpledata-3.1.0/docs/api/marple.db.DB.rst

@@ -0,0 +1,49 @@
+marple.db.DB
+============
+
+.. currentmodule:: marple.db
+
+.. autoclass:: DB
+
+   
+   .. automethod:: __init__
+
+   
+   .. rubric:: Methods
+
+   .. autosummary::
+   
+      ~DB.add_dataset
+      ~DB.check_connection
+      ~DB.create_stream
+      ~DB.dataset_append
+      ~DB.delete
+      ~DB.delete_dataset
+      ~DB.delete_stream
+      ~DB.download_original
+      ~DB.download_signal
+      ~DB.get
+      ~DB.get_dataset
+      ~DB.get_datasets
+      ~DB.get_signal
+      ~DB.get_signals
+      ~DB.get_status
+      ~DB.get_stream
+      ~DB.get_streams
+      ~DB.patch
+      ~DB.post
+      ~DB.push_file
+      ~DB.update_metadata
+      ~DB.upsert_signals
+   
+   
+
+   
+   
+   .. rubric:: Attributes
+
+   .. autosummary::
+   
+      ~DB.client
+   
+   

marpledata-3.1.0/docs/api/marple.db.DataStream.rst

@@ -0,0 +1,49 @@
+marple.db.DataStream
+====================
+
+.. currentmodule:: marple.db
+
+.. autoclass:: DataStream
+
+   
+   .. automethod:: __init__
+
+   
+   .. rubric:: Methods
+
+   .. autosummary::
+   
+      ~DataStream.delete
+      ~DataStream.get_dataset
+      ~DataStream.get_datasets
+      ~DataStream.push_file
+   
+   
+
+   
+   
+   .. rubric:: Attributes
+
+   .. autosummary::
+   
+      ~DataStream.type
+      ~DataStream.id
+      ~DataStream.name
+      ~DataStream.description
+      ~DataStream.datapool
+      ~DataStream.layer_shifts
+      ~DataStream.version_id
+      ~DataStream.insight_workspace
+      ~DataStream.insight_project
+      ~DataStream.created_at
+      ~DataStream.last_updated
+      ~DataStream.last_ingested
+      ~DataStream.n_datasets
+      ~DataStream.n_datapoints
+      ~DataStream.cold_bytes
+      ~DataStream.hot_bytes
+      ~DataStream.plugin
+      ~DataStream.plugin_args
+      ~DataStream.signal_reduction
+   
+   

marpledata-3.1.0/docs/api/marple.db.Dataset.rst

@@ -0,0 +1,58 @@
+marple.db.Dataset
+=================
+
+.. currentmodule:: marple.db
+
+.. autoclass:: Dataset
+
+   
+   .. automethod:: __init__
+
+   
+   .. rubric:: Methods
+
+   .. autosummary::
+   
+      ~Dataset.delete
+      ~Dataset.download
+      ~Dataset.fetch
+      ~Dataset.get_data
+      ~Dataset.get_signal
+      ~Dataset.get_signals
+      ~Dataset.update_metadata
+      ~Dataset.wait_for_import
+   
+   
+
+   
+   
+   .. rubric:: Attributes
+
+   .. autosummary::
+   
+      ~Dataset.id
+      ~Dataset.datastream_id
+      ~Dataset.datastream_version
+      ~Dataset.created_at
+      ~Dataset.created_by
+      ~Dataset.import_status
+      ~Dataset.import_progress
+      ~Dataset.import_message
+      ~Dataset.import_time
+      ~Dataset.path
+      ~Dataset.metadata
+      ~Dataset.cold_path
+      ~Dataset.cold_bytes
+      ~Dataset.hot_bytes
+      ~Dataset.backup_path
+      ~Dataset.backup_size
+      ~Dataset.plugin
+      ~Dataset.plugin_args
+      ~Dataset.n_datapoints
+      ~Dataset.n_signals
+      ~Dataset.timestamp_start
+      ~Dataset.timestamp_stop
+      ~Dataset.import_speed
+      ~Dataset.parquet_version
+   
+   

marpledata-3.1.0/docs/api/marple.db.DatasetList.rst

@@ -0,0 +1,40 @@
+marple.db.DatasetList
+=====================
+
+.. currentmodule:: marple.db
+
+.. autoclass:: DatasetList
+
+   
+   .. automethod:: __init__
+
+   
+   .. rubric:: Methods
+
+   .. autosummary::
+   
+      ~DatasetList.append
+      ~DatasetList.clear
+      ~DatasetList.count
+      ~DatasetList.extend
+      ~DatasetList.from_dicts
+      ~DatasetList.get_data
+      ~DatasetList.index
+      ~DatasetList.insert
+      ~DatasetList.pop
+      ~DatasetList.remove
+      ~DatasetList.reverse
+      ~DatasetList.sort
+      ~DatasetList.to_pandas
+      ~DatasetList.wait_for_import
+      ~DatasetList.where
+      ~DatasetList.where_dataset
+      ~DatasetList.where_imported
+      ~DatasetList.where_metadata
+      ~DatasetList.where_signal
+   
+   
+
+   
+   
+