jerry-thomas 1.0.3__py3-none-any.whl → 2.0.0__py3-none-any.whl

datapipeline/sources/decoders.py

@@ -1,12 +1,11 @@
-from __future__ import annotations
-
 from abc import ABC, abstractmethod
-from typing import Iterable, Iterator, Any, Optional
+from typing import Iterable, Iterator, Any, Optional, Sequence
 import codecs
 import csv
 import io
 import json
 import pickle
+import itertools
 
 
 class Decoder(ABC):
@@ -32,7 +31,7 @@ def _iter_text_lines(chunks: Iterable[bytes], encoding: str) -> Iterator[str]:
             idx = buffer.find("\n")
             if idx == -1:
                 break
-            line, buffer = buffer[:idx], buffer[idx + 1 :]
+            line, buffer = buffer[:idx], buffer[idx + 1:]
             if line.endswith("\r"):
                 line = line[:-1]
             yield line
@@ -53,26 +52,58 @@ def _read_all_text(chunks: Iterable[bytes], encoding: str) -> str:
 
 
 class CsvDecoder(Decoder):
-    def __init__(self, *, delimiter: str = ";", encoding: str = "utf-8"):
+    def __init__(
+        self,
+        *,
+        delimiter: str = ";",
+        encoding: str = "utf-8",
+        error_prefixes: Optional[Sequence[str]] = None,
+    ):
         self.delimiter = delimiter
         self.encoding = encoding
+        self._error_prefixes = [p.lower() for p in (error_prefixes or [])]
+
+    def _iter_lines(self, chunks: Iterable[bytes]) -> Iterator[str]:
+        lines = _iter_text_lines(chunks, self.encoding)
+        try:
+            first = next(lines)
+        except StopIteration:
+            return iter(())
+        if self._error_prefixes:
+            lowered = first.lstrip().lower()
+            if any(lowered.startswith(p) for p in self._error_prefixes):
+                raise ValueError(
+                    f"csv response looks like error text: {first[:120]}")
+        return itertools.chain([first], lines)
 
     def decode(self, chunks: Iterable[bytes]) -> Iterator[dict]:
-        reader = csv.DictReader(_iter_text_lines(chunks, self.encoding), delimiter=self.delimiter)
+        reader = csv.DictReader(self._iter_lines(
+            chunks), delimiter=self.delimiter)
         for row in reader:
             yield row
 
     def count(self, chunks: Iterable[bytes]) -> Optional[int]:
-        return sum(1 for _ in csv.DictReader(_iter_text_lines(chunks, self.encoding), delimiter=self.delimiter))
+        return sum(1 for _ in csv.DictReader(self._iter_lines(chunks), delimiter=self.delimiter))
 
 
 class JsonDecoder(Decoder):
-    def __init__(self, *, encoding: str = "utf-8"):
+    def __init__(self, *, encoding: str = "utf-8", array_field: Optional[str] = None):
         self.encoding = encoding
+        self.array_field = array_field
 
     def decode(self, chunks: Iterable[bytes]) -> Iterator[Any]:
         text = _read_all_text(chunks, self.encoding)
         data = json.loads(text)
+        if self.array_field:
+            if not isinstance(data, dict):
+                raise ValueError(
+                    "json array_field requires a top-level object")
+            if self.array_field not in data:
+                raise ValueError(
+                    f"json array_field missing: {self.array_field}")
+            data = data[self.array_field]
+            if data is None:
+                return  # TODO MAYBE we NEED DO DO SOMETHING ABOUT THIS so we dont silence it
         if isinstance(data, list):
             for item in data:
                 yield item
@@ -83,6 +114,16 @@ class JsonDecoder(Decoder):
     def count(self, chunks: Iterable[bytes]) -> Optional[int]:
         text = _read_all_text(chunks, self.encoding)
         data = json.loads(text)
+        if self.array_field:
+            if not isinstance(data, dict):
+                raise ValueError(
+                    "json array_field requires a top-level object")
+            if self.array_field not in data:
+                raise ValueError(
+                    f"json array_field missing: {self.array_field}")
+            data = data[self.array_field]
+            if data is None:
+                return 0
         return len(data) if isinstance(data, list) else 1
 
 

datapipeline/sources/factory.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from typing import Any, Dict
 
 from datapipeline.sources.data_loader import DataLoader
@@ -19,7 +17,9 @@ def build_loader(*, transport: str, format: str | None = None, **kwargs: Any) ->
       transport: "fs" | "http"
       format: "csv" | "json" | "json-lines" | "pickle" (required for fs/http)
       fs: path (str), glob (bool, optional), encoding (str, default utf-8), delimiter (csv only)
-      http: url (str), headers (dict, optional), params (dict, optional), encoding (str, default utf-8)
+      http: url (str), headers (dict, optional), params (dict, optional), encoding (str, default utf-8), timeout_seconds (float, optional)
+      csv: error_prefixes (list[str], optional)
+      json: array_field (str, optional)
     """
 
     t = (transport or "").lower()
@@ -40,16 +40,19 @@ def build_loader(*, transport: str, format: str | None = None, **kwargs: Any) ->
         headers: Dict[str, str] = dict(kwargs.get("headers") or {})
         params: Dict[str, Any] = dict(kwargs.get("params") or {})
         encoding = kwargs.get("encoding", "utf-8")
-        source = HttpTransport(url, headers=headers, params=params)
+        timeout_seconds = kwargs.get("timeout_seconds")
+        source = HttpTransport(url, headers=headers, params=params, timeout_seconds=timeout_seconds)
     else:
         raise ValueError(f"unsupported transport: {transport}")
 
     # Build decoder
     if fmt == "csv":
         delimiter = kwargs.get("delimiter", ";")
-        decoder = CsvDecoder(delimiter=delimiter, encoding=encoding)
+        error_prefixes = kwargs.get("error_prefixes")
+        decoder = CsvDecoder(delimiter=delimiter, encoding=encoding, error_prefixes=error_prefixes)
     elif fmt == "json":
-        decoder = JsonDecoder(encoding=encoding)
+        array_field = kwargs.get("array_field")
+        decoder = JsonDecoder(encoding=encoding, array_field=array_field)
     elif fmt == "json-lines":
         decoder = JsonLinesDecoder(encoding=encoding)
     elif fmt == "pickle":

datapipeline/sources/foreach.py

@@ -1,6 +1,5 @@
-from __future__ import annotations
-
 import re
+import time
 from typing import Any, Iterator, Mapping
 
 from datapipeline.plugins import LOADERS_EP
@@ -49,11 +48,13 @@ class ForeachLoader(BaseDataLoader):
         loader: Mapping[str, Any],
         inject_field: str | None = None,
         inject: Mapping[str, Any] | None = None,
+        throttle_seconds: float | None = None,
     ):
         self._key, self._values = self._normalize_foreach(foreach)
         self._loader_spec = self._normalize_loader_spec(loader)
         self._inject_field = inject_field
         self._inject = inject
+        self._throttle_seconds = self._normalize_throttle(throttle_seconds)
         self._current_index: int | None = None
         self._current_value: Any | None = None
         self._current_args: dict[str, Any] | None = None
@@ -68,6 +69,8 @@ class ForeachLoader(BaseDataLoader):
 
     def load(self) -> Iterator[Any]:
         for i, value in enumerate(self._values, 1):
+            if self._throttle_seconds and i > 1:
+                time.sleep(self._throttle_seconds)
             vars_ = {self._key: value}
             loader_args = self._make_loader_args(vars_)
             loader = self._build_loader(loader_args)
@@ -84,7 +87,9 @@ class ForeachLoader(BaseDataLoader):
 
     def count(self):
         total = 0
-        for value in self._values:
+        for i, value in enumerate(self._values, 1):
+            if self._throttle_seconds and i > 1:
+                time.sleep(self._throttle_seconds)
             vars_ = {self._key: value}
             loader_args = self._make_loader_args(vars_)
             loader = self._build_loader(loader_args)
@@ -119,6 +124,16 @@ class ForeachLoader(BaseDataLoader):
             raise TypeError("core.foreach loader.args must be a mapping when provided")
         return dict(loader)
 
+    @staticmethod
+    def _normalize_throttle(throttle_seconds: float | None) -> float:
+        if throttle_seconds is None:
+            return 0.0
+        if not isinstance(throttle_seconds, (int, float)):
+            raise TypeError("core.foreach throttle_seconds must be a number")
+        if throttle_seconds < 0:
+            raise ValueError("core.foreach throttle_seconds must be >= 0")
+        return float(throttle_seconds)
+
     def _make_loader_args(self, vars_: Mapping[str, Any]) -> dict[str, Any]:
         args = self._loader_spec.get("args") or {}
         interpolated = _interpolate(args, vars_)

datapipeline/sources/synthetic/time/parser.py

@@ -6,4 +6,4 @@ from datapipeline.domain.record import TemporalRecord
 class TimeRowParser(DataParser[TemporalRecord]):
     def parse(self, raw: Dict[str, Any]) -> Optional[TemporalRecord]:
         t = raw["time"]
-        return TemporalRecord(time=t, value=t)
+        return TemporalRecord(time=t)

datapipeline/sources/transports.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from abc import ABC, abstractmethod
 from typing import Iterable, Iterator, List, Dict, Optional, Any
 from urllib.request import Request, urlopen
@@ -65,11 +63,19 @@ class FsGlobTransport(Transport):
 
 
 class HttpTransport(Transport):
-    def __init__(self, url: str, headers: Optional[Dict[str, str]] = None, params: Optional[Dict[str, Any]] = None, chunk_size: int = 64 * 1024):
+    def __init__(
+        self,
+        url: str,
+        headers: Optional[Dict[str, str]] = None,
+        params: Optional[Dict[str, Any]] = None,
+        chunk_size: int = 64 * 1024,
+        timeout_seconds: Optional[float] = None,
+    ):
         self.url = url
         self.headers = dict(headers or {})
         self.params: Dict[str, Any] = dict(params or {})
         self.chunk_size = chunk_size
+        self.timeout_seconds = timeout_seconds
 
     def _build_url(self) -> str:
         if not self.params:
@@ -88,7 +94,7 @@ class HttpTransport(Transport):
         req = Request(req_url, headers=self.headers)
 
         try:
-            resp = urlopen(req)
+            resp = urlopen(req, timeout=self.timeout_seconds)
         except (URLError, HTTPError) as e:
             raise RuntimeError(f"failed to fetch {self.url}: {e}") from e
 

datapipeline/templates/demo_skeleton/demo/contracts/equity.ohlcv.yaml

@@ -0,0 +1,33 @@
+kind: ingest
+source: sandbox.ohlcv
+id: equity.ohlcv # format: domain.dataset.(variant)
+
+mapper:
+  entrypoint: map_sandbox_ohlcv_dto_to_equity
+  args: {}
+
+cadence: ${group_by} # optional per-contract cadence
+partition_by: ticker
+# sort_batch_size: 100000              # in-memory sort chunk size
+
+record: # record-level transforms
+  - filter: { field: time, operator: ge, comparand: "${start_time}" }
+  - filter: { field: time, operator: le, comparand: "${end_time}" }
+  - floor_time: { cadence: "${cadence}" }
+#   - lag: { lag: 10m }
+
+stream: # per-stream transforms (input sorted by partition,time)
+  # - ensure_cadence: { field: close, to: close, cadence: "${cadence}" }
+  # - granularity: { field: close, to: close, mode: first }
+  - rolling: {
+        field: dollar_volume,
+        to: adv5,
+        window: 5,
+        statistic: mean,
+        min_samples: 3,
+      } # compute 5-day average dollar volume (ADV5)
+  - filter: { field: adv5, operator: ge, comparand: 1_000_000 } # filter out illiquid stocks
+#   - fill: { statistic: median, window: 6, min_samples: 1 }
+
+debug: # optional validation-only checks
+  #- lint: { mode: warn, tick: "${cadence}" }

datapipeline/templates/demo_skeleton/demo/contracts/time.ticks.hour_sin.yaml

@@ -0,0 +1,22 @@
+# See ../../reference/reference/contracts/ingest.reference.yaml for full options.
+kind: ingest
+source: synthetic.ticks
+id: time.ticks.hour_sin
+cadence: ${group_by}
+
+mapper:
+  entrypoint: encode_time
+  args: { mode: hour_sin }
+
+record:
+  - filter: { field: time, operator: ge, comparand: "${start_time}" }
+  - filter: { field: time, operator: le, comparand: "${end_time}" }
+  - floor_time: { cadence: "${cadence}" }
+
+stream:
+  - dedupe: {}
+  - granularity: { field: value, to: value, mode: first }
+  - ensure_cadence: { field: value, to: value, cadence: "${cadence}" }
+
+debug:
+  - lint: { mode: error, tick: "${cadence}" }

datapipeline/templates/demo_skeleton/demo/contracts/time.ticks.linear.yaml

@@ -0,0 +1,22 @@
+# See ../../reference/reference/contracts/ingest.reference.yaml for full options.
+kind: ingest
+source: synthetic.ticks
+id: time.ticks.linear
+cadence: ${group_by}
+
+mapper:
+  entrypoint: encode_time
+  args: { mode: linear }
+
+record:
+  - filter: { field: time, operator: ge, comparand: "${start_time}" }
+  - filter: { field: time, operator: le, comparand: "${end_time}" }
+  - floor_time: { cadence: "${cadence}" }
+
+stream:
+  - dedupe: {}
+  - granularity: { field: value, to: value, mode: first }
+  - ensure_cadence: { field: value, to: value, cadence: "${cadence}" }
+
+debug:
+  - lint: { mode: error, tick: "${cadence}" }

datapipeline/templates/demo_skeleton/demo/data/APPL.jsonl

@@ -0,0 +1,19 @@
+{"time": "2021-01-04 05:00:00+00:00", "open": 129.99, "high": 130.06, "low": 123.47, "close": 126.15, "volume": 1549553.0, "symbol": "AAPL"}
+{"time": "2021-01-05 05:00:00+00:00", "open": 125.55, "high": 128.25, "low": 125.06, "close": 127.5, "volume": 804637.0, "symbol": "AAPL"}
+{"time": "2021-01-06 05:00:00+00:00", "open": 124.2, "high": 127.57, "low": 123.07, "close": 123.35, "volume": 2202534.0, "symbol": "AAPL"}
+{"time": "2021-01-07 05:00:00+00:00", "open": 124.98, "high": 128.14, "low": 124.51, "close": 127.42, "volume": 1440239.0, "symbol": "AAPL"}
+{"time": "2021-01-08 05:00:00+00:00", "open": 128.9, "high": 129.11, "low": 126.81, "close": 128.65, "volume": 1340001.0, "symbol": "AAPL"}
+{"time": "2021-01-11 05:00:00+00:00", "open": 125.81, "high": 126.67, "low": 125.1, "close": 125.53, "volume": 1168071.0, "symbol": "AAPL"}
+{"time": "2021-01-12 05:00:00+00:00", "open": 125.03, "high": 126.26, "low": 123.55, "close": 125.47, "volume": 810812.0, "symbol": "AAPL"}
+{"time": "2021-01-13 05:00:00+00:00", "open": 125.29, "high": 127.96, "low": 125.1, "close": 127.56, "volume": 1341043.0, "symbol": "AAPL"}
+{"time": "2021-01-14 05:00:00+00:00", "open": 127.31, "high": 127.53, "low": 125.39, "close": 125.69, "volume": 969718.0, "symbol": "AAPL"}
+{"time": "2021-01-15 05:00:00+00:00", "open": 125.27, "high": 126.76, "low": 123.65, "close": 123.95, "volume": 1159284.0, "symbol": "AAPL"}
+{"time": "2021-01-19 05:00:00+00:00", "open": 124.37, "high": 125.29, "low": 123.59, "close": 124.39, "volume": 1051940.0, "symbol": "AAPL"}
+{"time": "2021-01-20 05:00:00+00:00", "open": 125.16, "high": 128.99, "low": 125.16, "close": 128.52, "volume": 957072.0, "symbol": "AAPL"}
+{"time": "2021-01-21 05:00:00+00:00", "open": 130.25, "high": 133.7, "low": 130.06, "close": 133.19, "volume": 1994077.0, "symbol": "AAPL"}
+{"time": "2021-01-22 05:00:00+00:00", "open": 132.68, "high": 136.15, "low": 131.45, "close": 135.37, "volume": 1820717.0, "symbol": "AAPL"}
+{"time": "2021-01-25 05:00:00+00:00", "open": 139.22, "high": 141.2, "low": 132.94, "close": 139.14, "volume": 1957404.0, "symbol": "AAPL"}
+{"time": "2021-01-26 05:00:00+00:00", "open": 139.74, "high": 140.48, "low": 137.68, "close": 139.51, "volume": 1242288.0, "symbol": "AAPL"}
+{"time": "2021-01-27 05:00:00+00:00", "open": 139.62, "high": 140.44, "low": 136.79, "close": 138.26, "volume": 1965025.0, "symbol": "AAPL"}
+{"time": "2021-01-28 05:00:00+00:00", "open": 135.82, "high": 138.23, "low": 133.1, "close": 133.1, "volume": 2645618.0, "symbol": "AAPL"}
+{"time": "2021-01-29 05:00:00+00:00", "open": 132.29, "high": 133.09, "low": 126.79, "close": 128.3, "volume": 2609717.0, "symbol": "AAPL"}

datapipeline/templates/demo_skeleton/demo/data/MSFT.jsonl

@@ -0,0 +1,19 @@
+{"time": "2021-01-04 05:00:00+00:00", "open": 213.55, "high": 213.95, "low": 206.18, "close": 209.1, "volume": 678049.0, "symbol": "MSFT"}
+{"time": "2021-01-05 05:00:00+00:00", "open": 208.76, "high": 209.64, "low": 207.24, "close": 209.29, "volume": 483132.0, "symbol": "MSFT"}
+{"time": "2021-01-06 05:00:00+00:00", "open": 203.82, "high": 207.69, "low": 203.41, "close": 203.5, "volume": 881552.0, "symbol": "MSFT"}
+{"time": "2021-01-07 05:00:00+00:00", "open": 205.33, "high": 210.43, "low": 205.06, "close": 209.52, "volume": 621610.0, "symbol": "MSFT"}
+{"time": "2021-01-08 05:00:00+00:00", "open": 209.81, "high": 211.59, "low": 208.24, "close": 210.74, "volume": 656869.0, "symbol": "MSFT"}
+{"time": "2021-01-11 05:00:00+00:00", "open": 209.6, "high": 210.02, "low": 207.96, "close": 208.67, "volume": 519302.0, "symbol": "MSFT"}
+{"time": "2021-01-12 05:00:00+00:00", "open": 207.77, "high": 208.27, "low": 204.68, "close": 206.44, "volume": 705831.0, "symbol": "MSFT"}
+{"time": "2021-01-13 05:00:00+00:00", "open": 205.3, "high": 207.96, "low": 205.3, "close": 207.83, "volume": 635639.0, "symbol": "MSFT"}
+{"time": "2021-01-14 05:00:00+00:00", "open": 207.37, "high": 208.6, "low": 204.12, "close": 204.77, "volume": 573145.0, "symbol": "MSFT"}
+{"time": "2021-01-15 05:00:00+00:00", "open": 204.57, "high": 205.82, "low": 203.48, "close": 204.15, "volume": 669016.0, "symbol": "MSFT"}
+{"time": "2021-01-19 05:00:00+00:00", "open": 205.09, "high": 208.19, "low": 204.04, "close": 207.69, "volume": 688794.0, "symbol": "MSFT"}
+{"time": "2021-01-20 05:00:00+00:00", "open": 209.06, "high": 216.63, "low": 208.55, "close": 215.25, "volume": 1221985.0, "symbol": "MSFT"}
+{"time": "2021-01-21 05:00:00+00:00", "open": 215.56, "high": 217.12, "low": 213.43, "close": 215.79, "volume": 1226767.0, "symbol": "MSFT"}
+{"time": "2021-01-22 05:00:00+00:00", "open": 217.95, "high": 220.71, "low": 216.89, "close": 216.89, "volume": 1457906.0, "symbol": "MSFT"}
+{"time": "2021-01-25 05:00:00+00:00", "open": 219.82, "high": 220.46, "low": 215.26, "close": 220.09, "volume": 976700.0, "symbol": "MSFT"}
+{"time": "2021-01-26 05:00:00+00:00", "open": 222.66, "high": 224.68, "low": 220.77, "close": 223.1, "volume": 1510093.0, "symbol": "MSFT"}
+{"time": "2021-01-27 05:00:00+00:00", "open": 228.35, "high": 230.69, "low": 220.83, "close": 223.45, "volume": 2063324.0, "symbol": "MSFT"}
+{"time": "2021-01-28 05:00:00+00:00", "open": 226.06, "high": 232.79, "low": 225.58, "close": 229.35, "volume": 1701928.0, "symbol": "MSFT"}
+{"time": "2021-01-29 05:00:00+00:00", "open": 225.83, "high": 228.36, "low": 222.07, "close": 222.31, "volume": 1490894.0, "symbol": "MSFT"}

datapipeline/templates/demo_skeleton/demo/dataset.yaml

@@ -0,0 +1,19 @@
+# See ../reference/reference/dataset.yaml for full options.
+group_by: ${group_by}
+
+features:
+  - id: linear_time
+    record_stream: time.ticks.linear
+    field: value
+    scale: true
+    sequence: { size: 2, stride: 1 }
+
+  - id: closing_price
+    record_stream: equity.ohlcv
+    field: close
+    scale: true
+
+  - id: opening_price
+    record_stream: equity.ohlcv
+    field: open
+    scale: true

datapipeline/templates/demo_skeleton/demo/postprocess.yaml

@@ -0,0 +1,19 @@
+# See ../reference/reference/postprocess.yaml for full options.
+# - drop: # no targets so no effect but included here for demonstration
+#     axis: vertical
+#     payload: targets
+#     threshold: 0.9
+
+- drop: # effectively drops features with >50% missing values. 0 drops in the demo, but included here for demonstration
+    axis: vertical
+    payload: features
+    threshold: 0.5
+
+- drop: # this actually drops some vectors
+    axis: horizontal
+    payload: features
+    threshold: 1
+# - drop: # no targets so no effect but included here for demonstration
+#     axis: horizontal
+#     payload: targets
+#     threshold: 1

datapipeline/templates/demo_skeleton/demo/project.yaml

@@ -0,0 +1,19 @@
+# See ../reference/reference/project.yaml for full options.
+version: 1
+name: demo
+paths:
+  streams: ./contracts
+  sources: ./sources
+  dataset: dataset.yaml
+  postprocess: postprocess.yaml
+  artifacts: ../artifacts/${project_name}/v${version}
+  tasks: ./tasks
+globals:
+  group_by: 1d
+  start_time: 2021-01-01T00:00:00Z
+  end_time: 2021-02-01T00:00:00Z
+  split:
+    mode: hash
+    key: group
+    seed: 42
+    ratios: { train: 0.8, val: 0.1, test: 0.1 }

datapipeline/templates/demo_skeleton/demo/sources/sandbox.ohlcv.yaml

@@ -0,0 +1,17 @@
+# Required identifier for this raw source. Contracts reference it under `source:`.
+id: "sandbox.ohlcv" # suggested format: provider.dataset
+
+# parser.entrypoint: registered parser name (not a file path)
+parser:
+  entrypoint: "sandbox_ohlcv_dto_parser"
+  args: {}
+
+# loader.entrypoint: registered loader name (not a file path)
+loader:
+  entrypoint: "core.io"
+  args:
+    transport: fs
+    format: json-lines
+    path: demo/data/*.jsonl
+    glob: true
+    encoding: utf-8

datapipeline/templates/{plugin_skeleton/example → demo_skeleton/demo}/sources/synthetic.ticks.yaml

@@ -1,3 +1,4 @@
+# See ../../reference/reference/sources/overview.reference.yaml for full options.
 id: synthetic.ticks
 
 parser:
@@ -9,4 +10,3 @@ loader:
     start: "${start_time}"
     end: "${end_time}"
     frequency: "${group_by}"
-

datapipeline/templates/demo_skeleton/demo/tasks/metadata.yaml

@@ -0,0 +1,2 @@
+# See ../../reference/reference/tasks/metadata.reference.yaml for full options.
+kind: metadata

datapipeline/templates/demo_skeleton/demo/tasks/scaler.yaml

@@ -0,0 +1,3 @@
+# See ../../reference/reference/tasks/scaler.reference.yaml for full options.
+kind: scaler
+split_label: train

datapipeline/templates/demo_skeleton/demo/tasks/schema.yaml

@@ -0,0 +1,2 @@
+# See ../../reference/reference/tasks/schema.reference.yaml for full options.
+kind: schema

datapipeline/templates/demo_skeleton/demo/tasks/serve.test.yaml

@@ -0,0 +1,4 @@
+# See ../../reference/reference/tasks/serve.reference.yaml for full options.
+kind: serve
+name: test
+keep: test

datapipeline/templates/demo_skeleton/demo/tasks/serve.train.yaml

@@ -0,0 +1,4 @@
+# See ../../reference/reference/tasks/serve.reference.yaml for full options.
+kind: serve
+name: train
+keep: train

datapipeline/templates/demo_skeleton/demo/tasks/serve.val.yaml

@@ -0,0 +1,4 @@
+# See ../../reference/reference/tasks/serve.reference.yaml for full options.
+kind: serve
+name: val
+keep: val

datapipeline/templates/demo_skeleton/scripts/run_dataframe.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+from datapipeline.integrations import dataframe_from_vectors
+
+
+def main() -> None:
+    project = Path(__file__).resolve().parent / "project.yaml"
+    df = dataframe_from_vectors(
+        project,
+        limit=None,
+        include_group=True,
+        group_format="mapping",
+        flatten_sequences=True,
+    )
+    print("DataFrame shape:", df.shape)
+    print(df.head())
+
+
+if __name__ == "__main__":
+    main()

datapipeline/templates/demo_skeleton/scripts/run_torch.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+import torch
+from torch.utils.data import DataLoader
+
+from datapipeline.integrations import torch_dataset
+
+
+def main() -> None:
+    project = Path(__file__).resolve().parent / "project.yaml"
+    ds = torch_dataset(
+        project,
+        limit=256,
+        dtype=torch.float32,
+        flatten_sequences=True,
+    )
+    loader = DataLoader(ds, batch_size=32, shuffle=True)
+    batch = next(iter(loader))
+    print("Feature batch shape:", batch.shape)
+
+
+if __name__ == "__main__":
+    main()

datapipeline/templates/demo_skeleton/src/{{PACKAGE_NAME}}/domains/equity/model.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+from datapipeline.domain.record import TemporalRecord
+
+
+@dataclass
+class EquityRecord(TemporalRecord):
+    """
+    Domain record for 'equity'.
+    """
+    open: float
+    high: float
+    low: float
+    close: float
+    volume: float
+    dollar_volume: float
+    hl_range: float
+    ticker: str  # equity ticker symbol

datapipeline/templates/demo_skeleton/src/{{PACKAGE_NAME}}/dtos/sandbox_ohlcv_dto.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass
+class SandboxOhlcvDTO:
+    """Data Transfer Object (DTO) for sandbox OHLCV records."""
+    time: datetime
+    open: float
+    high: float
+    low: float
+    close: float
+    volume: float
+    symbol: str

datapipeline/templates/demo_skeleton/src/{{PACKAGE_NAME}}/mappers/map_sandbox_ohlcv_dto_to_equity.py

@@ -0,0 +1,26 @@
+from typing import Any, Iterator
+
+from {{PACKAGE_NAME}}.domains.equity.model import EquityRecord
+from {{PACKAGE_NAME}}.dtos.sandbox_ohlcv_dto import SandboxOhlcvDTO
+
+
+def map_sandbox_ohlcv_dto_to_equity(
+    stream: Iterator[SandboxOhlcvDTO],
+    **params: Any,
+) -> Iterator[EquityRecord]:
+    """Map SandboxOhlcvDTO records to domain-level EquityRecord records."""
+    for record in stream:
+        yield EquityRecord(
+            time=record.time,  # necessary for correct grouping and ordering
+
+            # filterable fields
+            open=record.open,
+            high=record.high,
+            low=record.low,
+            close=record.close,
+            volume=record.volume,
+            dollar_volume=record.close * record.volume,
+            hl_range=record.high - record.low,
+            ticker=record.symbol,
+            # filterable fields
+        )