pulse-data 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inventzia/pulse/data/__init__.py +19 -0
- inventzia/pulse/data/datum/__init__.py +29 -0
- inventzia/pulse/data/datum/codec.py +94 -0
- inventzia/pulse/data/datum/datum.py +50 -0
- inventzia/pulse/data/datum/provider.py +67 -0
- inventzia/pulse/data/datum/registry.py +269 -0
- inventzia/pulse/data/schemas/common/vector_value.py +56 -0
- inventzia/pulse/data/schemas/marketdata/cdf_bar.py +70 -0
- inventzia/pulse/data/schemas/platform/heart_beat.py +43 -0
- inventzia/pulse/data/schemas/platform/text_message.py +45 -0
- inventzia/pulse/data/schemas/provider.py +45 -0
- inventzia/pulse/data/schemas/registry.py +51 -0
- pulse_data-0.2.2.dist-info/METADATA +259 -0
- pulse_data-0.2.2.dist-info/RECORD +19 -0
- pulse_data-0.2.2.dist-info/WHEEL +5 -0
- pulse_data-0.2.2.dist-info/licenses/LICENSE-AGPL-3.0 +668 -0
- pulse_data-0.2.2.dist-info/licenses/LICENSE-COMMERCIAL.txt +22 -0
- pulse_data-0.2.2.dist-info/licenses/NOTICE +16 -0
- pulse_data-0.2.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Inventzia-Commercial
|
|
2
|
+
# Copyright (c) 2013-2026 Magrino Bini, Paola Apruzzese, Inventzia Science and Technology Ltd.
|
|
3
|
+
#
|
|
4
|
+
# This file is part of pulse-data.
|
|
5
|
+
#
|
|
6
|
+
# pulse-data is dual-licensed:
|
|
7
|
+
# - Under the GNU Affero General Public License v3.0 or later (see LICENSE-AGPL-3.0).
|
|
8
|
+
# - Under a commercial license (see LICENSE-COMMERCIAL.txt).
|
|
9
|
+
# Contact operations@inventzia.com.
|
|
10
|
+
#
|
|
11
|
+
# THIS FILE IS GENERATED. DO NOT EDIT MANUALLY.
|
|
12
|
+
# Source: schemas_yaml/platform/heartbeat.yaml
|
|
13
|
+
# Regenerate: python schemas/schemas-generators/generate_python.py
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
17
|
+
from typing import ClassVar, Optional
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class HeartBeat(BaseModel):
|
|
21
|
+
"""
|
|
22
|
+
Periodic platform heartbeat produced by HeartBeatGateway. Actors subscribe to a heartbeat Topic to implement periodic behaviour (analytics windows, timeout checks) that fires at regular simulation-time intervals regardless of whether domain events arrive in that interval.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
model_config = ConfigDict(extra="ignore", frozen=True)
|
|
26
|
+
|
|
27
|
+
TYPE_ID: ClassVar[str] = "com.inventzia.pulse.data.schemas.platform.HeartBeat"
|
|
28
|
+
TYPE_VERSION: ClassVar[int] = 1
|
|
29
|
+
|
|
30
|
+
beat_key: str = Field(alias="beatKey")
|
|
31
|
+
"""Heartbeat identifier. Typically a fixed label (e.g. "PERIODIC") or a group key when multiple independent heartbeat streams are needed"""
|
|
32
|
+
beat_time: int = Field(alias="beatTime")
|
|
33
|
+
"""Scheduled beat time in epoch milliseconds"""
|
|
34
|
+
|
|
35
|
+
# -- Datum protocol ---------------------------------------------------
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def datum_key(self) -> str:
|
|
39
|
+
return self.beat_key
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def datum_time(self) -> int:
|
|
43
|
+
return self.beat_time
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Inventzia-Commercial
|
|
2
|
+
# Copyright (c) 2013-2026 Magrino Bini, Paola Apruzzese, Inventzia Science and Technology Ltd.
|
|
3
|
+
#
|
|
4
|
+
# This file is part of pulse-data.
|
|
5
|
+
#
|
|
6
|
+
# pulse-data is dual-licensed:
|
|
7
|
+
# - Under the GNU Affero General Public License v3.0 or later (see LICENSE-AGPL-3.0).
|
|
8
|
+
# - Under a commercial license (see LICENSE-COMMERCIAL.txt).
|
|
9
|
+
# Contact operations@inventzia.com.
|
|
10
|
+
#
|
|
11
|
+
# THIS FILE IS GENERATED. DO NOT EDIT MANUALLY.
|
|
12
|
+
# Source: schemas_yaml/platform/text_message.yaml
|
|
13
|
+
# Regenerate: python schemas/schemas-generators/generate_python.py
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
17
|
+
from typing import ClassVar, Optional
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TextMessage(BaseModel):
|
|
21
|
+
"""
|
|
22
|
+
Generic carrier for a free-text string payload. The platform equivalent of a plain message — useful for diagnostics, echo/relay examples, and any actor or gateway that needs to move arbitrary text on a topic.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
model_config = ConfigDict(extra="ignore", frozen=True)
|
|
26
|
+
|
|
27
|
+
TYPE_ID: ClassVar[str] = "com.inventzia.pulse.data.schemas.platform.TextMessage"
|
|
28
|
+
TYPE_VERSION: ClassVar[int] = 1
|
|
29
|
+
|
|
30
|
+
msg_key: str = Field(alias="msgKey")
|
|
31
|
+
"""Routing key for this message — e.g. a channel, source identifier, or logical stream name"""
|
|
32
|
+
msg_time: int = Field(alias="msgTime")
|
|
33
|
+
"""Epoch milliseconds when the message was created"""
|
|
34
|
+
text: str
|
|
35
|
+
"""The free-text payload"""
|
|
36
|
+
|
|
37
|
+
# -- Datum protocol ---------------------------------------------------
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def datum_key(self) -> str:
|
|
41
|
+
return self.msg_key
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def datum_time(self) -> int:
|
|
45
|
+
return self.msg_time
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Inventzia-Commercial
|
|
2
|
+
# Copyright (c) 2013-2026 Magrino Bini, Paola Apruzzese, Inventzia Science and Technology Ltd.
|
|
3
|
+
#
|
|
4
|
+
# This file is part of pulse-data.
|
|
5
|
+
#
|
|
6
|
+
# pulse-data is dual-licensed:
|
|
7
|
+
# - Under the GNU Affero General Public License v3.0 or later (see LICENSE-AGPL-3.0).
|
|
8
|
+
# - Under a commercial license (see LICENSE-COMMERCIAL.txt).
|
|
9
|
+
# Contact operations@inventzia.com.
|
|
10
|
+
#
|
|
11
|
+
# THIS FILE IS GENERATED. DO NOT EDIT MANUALLY.
|
|
12
|
+
# Source: all schemas under schemas_yaml/
|
|
13
|
+
# Regenerate: python schemas/schemas-generators/generate_python.py
|
|
14
|
+
|
|
15
|
+
"""The datum-type provider (generated) for com.inventzia.pulse.data."""
|
|
16
|
+
|
|
17
|
+
from inventzia.pulse.data.datum.provider import DatumTypeBinding
|
|
18
|
+
from inventzia.pulse.data.schemas.common.vector_value import VectorValue
|
|
19
|
+
from inventzia.pulse.data.schemas.marketdata.cdf_bar import CdfBar
|
|
20
|
+
from inventzia.pulse.data.schemas.platform.heart_beat import HeartBeat
|
|
21
|
+
from inventzia.pulse.data.schemas.platform.text_message import TextMessage
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class CoreDatumTypeProvider:
|
|
25
|
+
"""Datum types contributed by com.inventzia.pulse.data, discovered via the SPI."""
|
|
26
|
+
|
|
27
|
+
def provider_id(self) -> str:
|
|
28
|
+
return "com.inventzia.pulse.data"
|
|
29
|
+
|
|
30
|
+
def spi_version(self) -> int:
|
|
31
|
+
return 1
|
|
32
|
+
|
|
33
|
+
def package_version(self) -> str:
|
|
34
|
+
return "0.2.2"
|
|
35
|
+
|
|
36
|
+
def bindings(self) -> "list[DatumTypeBinding]":
|
|
37
|
+
return [
|
|
38
|
+
DatumTypeBinding(VectorValue.TYPE_ID, VectorValue.TYPE_VERSION, VectorValue),
|
|
39
|
+
DatumTypeBinding(CdfBar.TYPE_ID, CdfBar.TYPE_VERSION, CdfBar),
|
|
40
|
+
DatumTypeBinding(HeartBeat.TYPE_ID, HeartBeat.TYPE_VERSION, HeartBeat),
|
|
41
|
+
DatumTypeBinding(TextMessage.TYPE_ID, TextMessage.TYPE_VERSION, TextMessage),
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
def manifest(self):
|
|
45
|
+
return "pdm1|com.inventzia.pulse.data|com.inventzia.pulse.data.schemas.common.VectorValue:1:47a0adfbbf19dcd7614cdf985086a67531a2bc9f5d1d6c5a5a721e6ee753ba23;com.inventzia.pulse.data.schemas.marketdata.CdfBar:1:3e6e7e76fc16188671c3c7d8036490b6c78bebcfa9293998af6d286e1a0dabd8;com.inventzia.pulse.data.schemas.platform.HeartBeat:1:5e08970887ce4d3424a8228b3a8b83622ba72a50d628ea155ca390802451844c;com.inventzia.pulse.data.schemas.platform.TextMessage:1:bb0ccb451f6abf888d58d3bab5bd82fbf31feab607139de5db38644977f1383a"
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Inventzia-Commercial
|
|
2
|
+
# Copyright (c) 2013-2026 Magrino Bini, Paola Apruzzese, Inventzia Science and Technology Ltd.
|
|
3
|
+
#
|
|
4
|
+
# This file is part of pulse-data.
|
|
5
|
+
#
|
|
6
|
+
# pulse-data is dual-licensed:
|
|
7
|
+
# - Under the GNU Affero General Public License v3.0 or later (see LICENSE-AGPL-3.0).
|
|
8
|
+
# - Under a commercial license (see LICENSE-COMMERCIAL.txt).
|
|
9
|
+
# Contact operations@inventzia.com.
|
|
10
|
+
#
|
|
11
|
+
# THIS FILE IS GENERATED. DO NOT EDIT MANUALLY.
|
|
12
|
+
# Source: all schemas under schemas_yaml/
|
|
13
|
+
# Regenerate: python schemas/schemas-generators/generate_python.py
|
|
14
|
+
|
|
15
|
+
"""Self-describing decode support: TYPE_ID -> generated model class."""
|
|
16
|
+
|
|
17
|
+
from inventzia.pulse.data.schemas.marketdata.cdf_bar import CdfBar
|
|
18
|
+
from inventzia.pulse.data.schemas.platform.heart_beat import HeartBeat
|
|
19
|
+
from inventzia.pulse.data.schemas.platform.text_message import TextMessage
|
|
20
|
+
from inventzia.pulse.data.schemas.common.vector_value import VectorValue
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
REGISTRY: dict[str, type] = {
|
|
24
|
+
CdfBar.TYPE_ID: CdfBar,
|
|
25
|
+
HeartBeat.TYPE_ID: HeartBeat,
|
|
26
|
+
TextMessage.TYPE_ID: TextMessage,
|
|
27
|
+
VectorValue.TYPE_ID: VectorValue,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def class_for(type_id: str) -> type:
|
|
32
|
+
"""Return the model class registered for a TYPE_ID."""
|
|
33
|
+
try:
|
|
34
|
+
return REGISTRY[type_id]
|
|
35
|
+
except KeyError:
|
|
36
|
+
raise KeyError(f"Unknown TYPE_ID: {type_id!r}") from None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def type_id_of(datum) -> str:
|
|
40
|
+
"""Return the TYPE_ID of a datum, verified against the registry.
|
|
41
|
+
|
|
42
|
+
Encoding must not emit a tagged envelope for a class that is not the
|
|
43
|
+
registered binding for its declared TYPE_ID, or a receiver could get a
|
|
44
|
+
typeId no runtime can decode. Mirrors the Java DatumTypeRegistry check.
|
|
45
|
+
"""
|
|
46
|
+
cls = type(datum)
|
|
47
|
+
type_id = getattr(cls, "TYPE_ID", None)
|
|
48
|
+
if REGISTRY.get(type_id) is not cls:
|
|
49
|
+
raise KeyError(
|
|
50
|
+
f"Unregistered datum type: {cls.__name__} (TYPE_ID {type_id!r})")
|
|
51
|
+
return type_id
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pulse-data
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: YAML-sourced data schemas and the Datum routing contract for the Pulse platform (one namespace, Java + Python).
|
|
5
|
+
Author-email: Magrino Bini <operations@inventzia.com>, Paola Apruzzese <operations@inventzia.com>
|
|
6
|
+
Maintainer-email: "Inventzia Science and Technology Ltd." <operations@inventzia.com>
|
|
7
|
+
License-Expression: AGPL-3.0-or-later OR LicenseRef-Inventzia-Commercial
|
|
8
|
+
Project-URL: Homepage, https://inventzia.com
|
|
9
|
+
Project-URL: Repository, https://github.com/inventzia-sci-tech/pulse-data
|
|
10
|
+
Project-URL: Changelog, https://github.com/inventzia-sci-tech/pulse-data/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: pulse,schemas,pydantic,algorithmic-trading,event-driven
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE-AGPL-3.0
|
|
21
|
+
License-File: LICENSE-COMMERCIAL.txt
|
|
22
|
+
License-File: NOTICE
|
|
23
|
+
Requires-Dist: pydantic<3,>=2
|
|
24
|
+
Provides-Extra: generator
|
|
25
|
+
Requires-Dist: pyyaml>=6; extra == "generator"
|
|
26
|
+
Requires-Dist: datamodel-code-generator>=0.25; extra == "generator"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# pulse-data
|
|
30
|
+
|
|
31
|
+
**pulse-data** is the single source of truth for the data types that flow across the Pulse
|
|
32
|
+
platform. Every event type is defined once in YAML and code-generated into strongly-typed,
|
|
33
|
+
immutable classes for both Java and Python. There is no hand-written serialisation code and no
|
|
34
|
+
way for the two language bindings to drift apart — they are projections of the same schema.
|
|
35
|
+
|
|
36
|
+
A consumer of pulse-data gets two things: a small, stable **routing contract** (`Datum`) that the
|
|
37
|
+
transport layer depends on, and a growing set of **generated data classes** that implement it.
|
|
38
|
+
|
|
39
|
+
This repository is deliberately narrow. It contains *only* the data definition — the contract,
|
|
40
|
+
the schemas, and the generators. It pulls no pandas, no SQLAlchemy, no Airflow. Anything that
|
|
41
|
+
*uses* the data (ingestion pipelines, storage backends, orchestration, shared helpers) lives in
|
|
42
|
+
**[pulse-utils](https://github.com/inventzia-sci-tech/pulse-utils)** and depends on pulse-data,
|
|
43
|
+
never the other way round.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## What lives here
|
|
48
|
+
|
|
49
|
+
| Area | Path | What it is |
|
|
50
|
+
|------|------|-----------|
|
|
51
|
+
| Routing contract | `datum/` | Hand-written `Datum` interface (Java) and `Protocol` (Python). The stable API everything else conforms to. |
|
|
52
|
+
| JSON serializer | `datum/` | `DatumCodec` (Java) and `datum/codec.py` (Python) — the canonical, shared serializer for `Datum` types (see below). |
|
|
53
|
+
| Schema sources | `schemas/schemas_yaml/` | YAML definitions — the single source of truth. |
|
|
54
|
+
| Generated Java | `schemas/schemas_java/` | Immutable Java `record` classes under `com.inventzia.pulse.data.schemas`. Build artefact; do not edit. |
|
|
55
|
+
| Generated Python | `src/inventzia/pulse/data/schemas/` | Pydantic v2 models under `inventzia.pulse.data.schemas` (mirrors Java), in the installable `src/` tree. Build artefact; do not edit. |
|
|
56
|
+
| Type registry | both | Composite `DatumTypeRegistry` (Java) / `datum/registry.py` (Python): TYPE_ID ↔ class, built from the core provider plus discovered extension providers (the `DatumTypeProvider` SPI), for self-describing decode. |
|
|
57
|
+
| Generators | `schemas/schemas-generators/` | `generate_java.py`, `generate_python.py`. |
|
|
58
|
+
|
|
59
|
+
Everything here is light: the Java side compiles to a small jar (Jackson + JSpecify only); the
|
|
60
|
+
Python side needs just PyYAML, datamodel-code-generator, and Pydantic.
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## Core idea: schema → routing-aware data class
|
|
65
|
+
|
|
66
|
+
A schema is plain JSON-Schema (draft-07) in YAML, with two pulse-specific annotations that mark
|
|
67
|
+
which fields carry routing meaning:
|
|
68
|
+
|
|
69
|
+
```yaml
|
|
70
|
+
# schemas/schemas_yaml/marketdata/cdf_bar.yaml
|
|
71
|
+
$id: "com.inventzia.pulse.data.schemas.marketdata.CdfBar"
|
|
72
|
+
title: CdfBar
|
|
73
|
+
type: object
|
|
74
|
+
properties:
|
|
75
|
+
symb:
|
|
76
|
+
type: string
|
|
77
|
+
x-datum-key: true # this field is the routing key
|
|
78
|
+
timestamp:
|
|
79
|
+
type: integer
|
|
80
|
+
format: int64
|
|
81
|
+
x-datum-time: true # this field is the logical event time
|
|
82
|
+
op: { type: number, format: decimal }
|
|
83
|
+
# ... more fields ...
|
|
84
|
+
required: [symb, timestamp, op]
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
From this, the generators produce classes that implement the `Datum` contract by delegating to the
|
|
88
|
+
annotated fields:
|
|
89
|
+
|
|
90
|
+
**Java** (`schemas_java/.../CdfBar.java`) — an immutable record:
|
|
91
|
+
```java
|
|
92
|
+
public record CdfBar(String symb, long timestamp, BigDecimal op /* ... */)
|
|
93
|
+
implements Datum {
|
|
94
|
+
public static final String TYPE_ID = "com.inventzia.pulse.data.schemas.marketdata.CdfBar";
|
|
95
|
+
public static final int TYPE_VERSION = 1;
|
|
96
|
+
@Override public String getDatumKey() { return symb; }
|
|
97
|
+
@Override public long getDatumTime() { return timestamp; }
|
|
98
|
+
}
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
**Python** (`src/inventzia/pulse/data/schemas/marketdata/cdf_bar.py`) — a Pydantic v2 model
|
|
102
|
+
satisfying the `Datum` protocol:
|
|
103
|
+
```python
|
|
104
|
+
class CdfBar(BaseModel):
|
|
105
|
+
TYPE_ID: ClassVar[str] = "com.inventzia.pulse.data.schemas.marketdata.CdfBar"
|
|
106
|
+
TYPE_VERSION: ClassVar[int] = 1
|
|
107
|
+
symb: str
|
|
108
|
+
timestamp: int
|
|
109
|
+
# ...
|
|
110
|
+
@property
|
|
111
|
+
def datum_key(self) -> str: return self.symb
|
|
112
|
+
@property
|
|
113
|
+
def datum_time(self) -> int: return self.timestamp
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
## Design decisions
|
|
119
|
+
|
|
120
|
+
- **YAML is the only source of truth.** Field definitions exist in exactly one place. Java and
|
|
121
|
+
Python are generated, never hand-edited. Regenerate after every schema change.
|
|
122
|
+
|
|
123
|
+
- **`$id` is the type identity.** Each schema's `$id` is the fully-qualified class name and becomes
|
|
124
|
+
the `TYPE_ID` constant in both languages — the single discriminator used for routing and
|
|
125
|
+
deserialisation. There is no separate numeric id to keep in sync.
|
|
126
|
+
|
|
127
|
+
- **The `Datum` contract is minimal.** Two methods — `getDatumKey()` and `getDatumTime()` — are all
|
|
128
|
+
the transport layer needs to route and time-order data. The schema author decides which domain
|
|
129
|
+
fields fulfil them via `x-datum-key` / `x-datum-time`. Nothing else about the payload is exposed
|
|
130
|
+
to the infrastructure.
|
|
131
|
+
|
|
132
|
+
- **Data classes are genuinely immutable.** Java records take defensive unmodifiable copies of any
|
|
133
|
+
list fields and reject null required fields in their compact constructor; Pydantic models are
|
|
134
|
+
`frozen` with tuple (not list) sequences. A value on the bus cannot be mutated by one consumer and
|
|
135
|
+
observed changed by another, nor become invalid after construction.
|
|
136
|
+
|
|
137
|
+
- **JSON is the wire format.** Java (Jackson) and Python (Pydantic) serialise to and from the same
|
|
138
|
+
JSON, so a value produced in one language is consumed verbatim in the other.
|
|
139
|
+
|
|
140
|
+
- **Serialization is owned here, not by callers.** Because pulse-data owns the types and schemas,
|
|
141
|
+
it also owns how a `Datum` becomes JSON. `DatumCodec` is the canonical serializer, a shared
|
|
142
|
+
singleton (`DatumCodec.instance()`), with `toJson(Datum)` / `fromJson(json, type)` as its whole
|
|
143
|
+
surface. It configures the JSON policy *once* for every field type the schemas use — JSR-310
|
|
144
|
+
`java.time` written as ISO-8601, `BigDecimal` for exact decimals, unknown properties ignored on
|
|
145
|
+
read for forward compatibility — and hides the underlying engine (Jackson) entirely. Consumers
|
|
146
|
+
never construct or pass a JSON mapper; downstream code (e.g. pulse-beacon's gateways) uses the
|
|
147
|
+
singleton and never references Jackson.
|
|
148
|
+
|
|
149
|
+
Two forms are offered. **Type-directed** (`toJson` / `fromJson(json, Class)`) when the caller
|
|
150
|
+
knows the concrete class — e.g. it knows a topic's payload type. **Self-describing** (`toTaggedJson`
|
|
151
|
+
/ `fromTaggedJson`) wraps the value in an envelope `{"typeId": "<TYPE_ID>", "payload": {…}}`, so a
|
|
152
|
+
receiver can recover the type from the message itself wherever it isn't known ahead of time — the
|
|
153
|
+
in-process cross-language bridge, and later the socket/ZMQ transport. The class is resolved through
|
|
154
|
+
a composite `TYPE_ID → class` registry (`DatumTypeRegistry` in Java, `datum/registry.py` in
|
|
155
|
+
Python), built from the core provider plus any discovered extension providers (the
|
|
156
|
+
`DatumTypeProvider` SPI). The same envelope is produced and consumed identically in both languages.
|
|
157
|
+
|
|
158
|
+
- **Forward compatibility is symmetric across languages.** Both bindings tolerate unknown fields on
|
|
159
|
+
read, so a newer producer that adds a field does not break an older consumer in either language:
|
|
160
|
+
Java disables `FAIL_ON_UNKNOWN_PROPERTIES`, and generated Python models use
|
|
161
|
+
`model_config(extra="ignore")`. This only covers *additive* changes; removals, renames, and type
|
|
162
|
+
changes are breaking and must bump `TYPE_VERSION`, so evolution within a major version is
|
|
163
|
+
additive-only by discipline. Because "ignore" silently drops drift (it would hide, say, one side
|
|
164
|
+
emitting a field the other does not), a cross-language round-trip test
|
|
165
|
+
(`pulse-beacon .../tests/test_codec_cross_language_parity.py`) asserts the tagged envelope is
|
|
166
|
+
identical after a Java↔Python round-trip — the loud drift detector `extra="forbid"` used to give.
|
|
167
|
+
|
|
168
|
+
- **Python uses structural typing.** The Python `Datum` is a `Protocol`; generated models satisfy
|
|
169
|
+
it by exposing `datum_key` / `datum_time` properties — no inheritance, no import coupling.
|
|
170
|
+
|
|
171
|
+
---
|
|
172
|
+
|
|
173
|
+
## Adding a new data type
|
|
174
|
+
|
|
175
|
+
### In pulse-data itself (a core type)
|
|
176
|
+
|
|
177
|
+
1. Create a YAML schema under `schemas/schemas_yaml/<area>/`, with a unique `$id`, a `title`
|
|
178
|
+
(becomes the class name), and exactly one `x-datum-key` and one `x-datum-time` field.
|
|
179
|
+
2. Regenerate both bindings:
|
|
180
|
+
```bash
|
|
181
|
+
cd schemas/schemas-generators
|
|
182
|
+
conda run -n pulse python generate_java.py
|
|
183
|
+
conda run -n pulse python generate_python.py
|
|
184
|
+
```
|
|
185
|
+
3. Commit the YAML **and** the regenerated Java/Python output.
|
|
186
|
+
|
|
187
|
+
See [`schemas/schemas-generators/readme.md`](https://github.com/inventzia-sci-tech/pulse-data/blob/main/schemas/schemas-generators/readme.md) for generator
|
|
188
|
+
options (paths, dry-run, verbose).
|
|
189
|
+
|
|
190
|
+
### From another package (an extension, via the `DatumTypeProvider` SPI)
|
|
191
|
+
|
|
192
|
+
Pulse ships a fixed set of core datum types (market bars, heartbeats, text messages), but an
|
|
193
|
+
adopter's domain rarely fits those alone. Say an IoT company adopts Pulse as the event-driven
|
|
194
|
+
transport layer for its device fleet: it needs to move its own specialized payloads (sensor
|
|
195
|
+
telemetry, device state, actuator commands), each with fields specific to its hardware. Because
|
|
196
|
+
every Pulse event is routed across the Java engine and the Python strategies and must decode
|
|
197
|
+
identically on both sides, such a payload cannot be an ad-hoc class in one language; it has to be a
|
|
198
|
+
first-class, self-describing `Datum` that both runtimes agree on. The SPI is how a downstream
|
|
199
|
+
package adds exactly that, on its own release cadence, without forking or modifying pulse-data.
|
|
200
|
+
|
|
201
|
+
A package contributes its custom `Datum` types by implementing the `DatumTypeProvider` Service
|
|
202
|
+
Provider Interface and declaring the types it adds in that provider's `bindings()` (each a type id,
|
|
203
|
+
version, and `Datum` class). The implementation is registered declaratively via a
|
|
204
|
+
`META-INF/services/…DatumTypeProvider` file bundled in the package's jar (and, in Python, an
|
|
205
|
+
`inventzia.pulse.datum_types` entry point). At runtime these providers are discovered by
|
|
206
|
+
`java.util.ServiceLoader`, which scans every jar on the classpath for that registration and loads
|
|
207
|
+
the listed provider classes. The discovery is performed once, explicitly, during engine
|
|
208
|
+
initialization (`DatumTypeRegistry.discoverProviders()`), which seeds the core provider and merges
|
|
209
|
+
in the discovered extensions into a single validated, frozen registry.
|
|
210
|
+
|
|
211
|
+
The extension owns its schema and generated bindings (it runs the same generators with its own
|
|
212
|
+
`--provider-id`/`--provider-class`), so pulse-data and pulse-beacon are never edited to add the
|
|
213
|
+
type. See the worked, runnable example at `examples/pulse-ext-example` (in pulse-beacon), which
|
|
214
|
+
defines an `ExtendedBar` datum and flows it through the engine end to end in both languages.
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## Environment
|
|
219
|
+
|
|
220
|
+
The Python generators run in the minimal conda environment defined by
|
|
221
|
+
[`py_environment.yml`](https://github.com/inventzia-sci-tech/pulse-data/blob/main/py_environment.yml):
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
conda env create -f py_environment.yml
|
|
225
|
+
conda activate pulse
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
This creates the shared `pulse` env (base layer). Working across the stack? pulse-beacon enriches
|
|
229
|
+
the same env with a JDK + Maven + the JPype bridge — see its README.
|
|
230
|
+
|
|
231
|
+
Java sources (the `Datum` interface and generated records) build with Maven via
|
|
232
|
+
[`pom.xml`](https://github.com/inventzia-sci-tech/pulse-data/blob/main/pom.xml); Java 17+.
|
|
233
|
+
|
|
234
|
+
---
|
|
235
|
+
|
|
236
|
+
## Licensing
|
|
237
|
+
|
|
238
|
+
Dual-licensed:
|
|
239
|
+
|
|
240
|
+
- **Open Source (AGPL v3.0 or later)** — see [`LICENSE-AGPL-3.0`](https://github.com/inventzia-sci-tech/pulse-data/blob/main/LICENSE-AGPL-3.0).
|
|
241
|
+
- **Commercial License** — see [`COMMERCIAL.md`](https://github.com/inventzia-sci-tech/pulse-data/blob/main/COMMERCIAL.md) for the summary and
|
|
242
|
+
[`LICENSE-COMMERCIAL.txt`](https://github.com/inventzia-sci-tech/pulse-data/blob/main/LICENSE-COMMERCIAL.txt) for the binding terms.
|
|
243
|
+
|
|
244
|
+
Contact operations@inventzia.com for commercial licensing.
|
|
245
|
+
|
|
246
|
+
## Contributing
|
|
247
|
+
|
|
248
|
+
By submitting a contribution you agree to [`CLA.md`](https://github.com/inventzia-sci-tech/pulse-data/blob/main/CLA.md), including the Developer Certificate
|
|
249
|
+
of Origin sign-off and the dual-licensing grant. CI enforces DCO sign-off on every PR commit.
|
|
250
|
+
|
|
251
|
+
## Security
|
|
252
|
+
|
|
253
|
+
Report vulnerabilities privately per [`SECURITY.md`](https://github.com/inventzia-sci-tech/pulse-data/blob/main/SECURITY.md). Do not open public issues for
|
|
254
|
+
security problems.
|
|
255
|
+
|
|
256
|
+
## Trademarks
|
|
257
|
+
|
|
258
|
+
"Pulse" and "Inventzia" are trademarks of Inventzia Science and Technology Ltd. The licenses for
|
|
259
|
+
this software do not grant any rights to use these trademarks.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
inventzia/pulse/data/__init__.py,sha256=97IR_BdtQ5gtjdHygDlPm1mBoHiNW2hULuzcKgk0TOM,965
|
|
2
|
+
inventzia/pulse/data/datum/__init__.py,sha256=iOnMP53RZL6yqsUhFMCAT1S6gdU17wozYwDVfp0uVi0,866
|
|
3
|
+
inventzia/pulse/data/datum/codec.py,sha256=kSIqxn8MlmbFY0H05WbKNu9Jb5gIYejSiLZQml4hC7o,3956
|
|
4
|
+
inventzia/pulse/data/datum/datum.py,sha256=GRJL0tdXjHIgf0izpnZc3jsBf9kmLDAZKU_8zdErhQA,1769
|
|
5
|
+
inventzia/pulse/data/datum/provider.py,sha256=msBbYF14oDIQRQbBe5wLYwFwWgxACyASvmODXCYQoR0,2564
|
|
6
|
+
inventzia/pulse/data/datum/registry.py,sha256=DpFB3duuVGAfO5y7wFCQfvI2r36OBWKSOKHkoOBeOog,11826
|
|
7
|
+
inventzia/pulse/data/schemas/provider.py,sha256=GFyAQ67McluvrzCQIfMgwEXhe8PvELdikm9cA13TwdM,2297
|
|
8
|
+
inventzia/pulse/data/schemas/registry.py,sha256=onLG10Wtzz7IJoenikLnlGaJm6WB4PI5A8GD_OR1W0c,1916
|
|
9
|
+
inventzia/pulse/data/schemas/common/vector_value.py,sha256=FLhUsSI6sq46yKu7vwD1AXXRUPtarODzfateL5ksV8U,2132
|
|
10
|
+
inventzia/pulse/data/schemas/marketdata/cdf_bar.py,sha256=uJj4N_KEpZL_pgbBA3dBCdwhPDZnAhoDTFm2Zd4ynCs,2213
|
|
11
|
+
inventzia/pulse/data/schemas/platform/heart_beat.py,sha256=FDgdRlL3tYQoOIZm8l69atnYk_jEDp2rD_70tPI7LHo,1748
|
|
12
|
+
inventzia/pulse/data/schemas/platform/text_message.py,sha256=bdUpBiYRfk6OAsaY1i1j04DQPUdDxysmMaM-joGNfU8,1696
|
|
13
|
+
pulse_data-0.2.2.dist-info/licenses/LICENSE-AGPL-3.0,sha256=wus861f4rHEuwRIuzANysgaYQlN-aJRNYGyYQ-oun7o,34824
|
|
14
|
+
pulse_data-0.2.2.dist-info/licenses/LICENSE-COMMERCIAL.txt,sha256=iRXI0mDkIpAxxjAS8Yk6iBpAOqEjwp1C991m48KBR1c,897
|
|
15
|
+
pulse_data-0.2.2.dist-info/licenses/NOTICE,sha256=x7uiR8sNQuv3NqQkTDfzl5znhQfKLqZz3uefNrb4ruc,695
|
|
16
|
+
pulse_data-0.2.2.dist-info/METADATA,sha256=xJ_vJB10YjpUPEkX2K4Va0CqW9t3c-zdHLEhKBXMtEA,13748
|
|
17
|
+
pulse_data-0.2.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
18
|
+
pulse_data-0.2.2.dist-info/top_level.txt,sha256=xEqvAhkOXMpZYeRA8teLRiLFkleq7xBq3GOHCuruin8,10
|
|
19
|
+
pulse_data-0.2.2.dist-info/RECORD,,
|