databento 0.87.0__tar.gz → 0.89.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {databento-0.87.0 → databento-0.89.0}/PKG-INFO +52 -26
- {databento-0.87.0 → databento-0.89.0}/README.md +35 -9
- {databento-0.87.0 → databento-0.89.0}/databento/common/dbnstore.py +6 -19
- {databento-0.87.0 → databento-0.89.0}/databento/common/parsing.py +2 -3
- {databento-0.87.0 → databento-0.89.0}/databento/common/symbology.py +2 -3
- databento-0.89.0/databento/common/zstd.py +255 -0
- {databento-0.87.0 → databento-0.89.0}/databento/historical/api/batch.py +7 -11
- {databento-0.87.0 → databento-0.89.0}/databento/historical/api/metadata.py +2 -2
- {databento-0.87.0 → databento-0.89.0}/databento/live/client.py +82 -1
- {databento-0.87.0 → databento-0.89.0}/databento/live/gateway.py +14 -0
- {databento-0.87.0 → databento-0.89.0}/databento/live/protocol.py +58 -0
- {databento-0.87.0 → databento-0.89.0}/databento/live/session.py +95 -22
- databento-0.89.0/databento/version.py +1 -0
- databento-0.89.0/pyproject.toml +90 -0
- databento-0.87.0/pyproject.toml → databento-0.89.0/pyproject.toml.orig +37 -31
- databento-0.87.0/databento/version.py +0 -1
- {databento-0.87.0 → databento-0.89.0}/LICENSE +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/__init__.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/__init__.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/bentologging.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/constants.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/cram.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/enums.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/error.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/http.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/iterator.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/publishers.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/system.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/types.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/common/validation.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/historical/__init__.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/historical/api/__init__.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/historical/api/symbology.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/historical/api/timeseries.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/historical/client.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/live/__init__.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/py.typed +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/reference/__init__.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/reference/api/__init__.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/reference/api/adjustment.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/reference/api/corporate.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/reference/api/security.py +0 -0
- {databento-0.87.0 → databento-0.89.0}/databento/reference/client.py +0 -0
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: databento
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.89.0
|
|
4
4
|
Summary: Official Python client library for Databento
|
|
5
|
+
Author: Databento
|
|
6
|
+
Author-email: Databento <support@databento.com>
|
|
5
7
|
License-Expression: Apache-2.0
|
|
6
8
|
License-File: LICENSE
|
|
7
|
-
Author: Databento
|
|
8
|
-
Author-email: support@databento.com
|
|
9
|
-
Requires-Python: >=3.10
|
|
10
9
|
Classifier: Development Status :: 4 - Beta
|
|
11
10
|
Classifier: Operating System :: OS Independent
|
|
12
11
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -19,20 +18,22 @@ Classifier: Topic :: Office/Business :: Financial
|
|
|
19
18
|
Classifier: Topic :: Office/Business :: Financial :: Investment
|
|
20
19
|
Classifier: Topic :: Software Development :: Libraries
|
|
21
20
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
-
Requires-Dist: aiohttp
|
|
23
|
-
Requires-Dist: aiohttp
|
|
24
|
-
Requires-Dist:
|
|
25
|
-
Requires-Dist:
|
|
26
|
-
Requires-Dist: numpy
|
|
27
|
-
Requires-Dist:
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
Requires-Dist:
|
|
30
|
-
Requires-Dist:
|
|
31
|
-
Requires-Dist:
|
|
32
|
-
|
|
33
|
-
|
|
21
|
+
Requires-Dist: aiohttp>=3.8.3,<4.0.0 ; python_full_version < '3.12'
|
|
22
|
+
Requires-Dist: aiohttp>=3.9.0,<4.0.0 ; python_full_version >= '3.12'
|
|
23
|
+
Requires-Dist: certifi>=2026.10.19
|
|
24
|
+
Requires-Dist: databento-dbn~=0.70.0
|
|
25
|
+
Requires-Dist: numpy>=1.23.5 ; python_full_version < '3.12'
|
|
26
|
+
Requires-Dist: numpy>=1.26.0 ; python_full_version >= '3.12'
|
|
27
|
+
Requires-Dist: pandas>=1.5.3,<4.0.0
|
|
28
|
+
Requires-Dist: pip-system-certs>=4.0 ; sys_platform == 'win32'
|
|
29
|
+
Requires-Dist: pyarrow>=13.0.0
|
|
30
|
+
Requires-Dist: requests>=2.27.0
|
|
31
|
+
Requires-Dist: zstandard>=0.21.0 ; python_full_version < '3.14'
|
|
32
|
+
Requires-Python: >=3.10
|
|
34
33
|
Project-URL: Homepage, https://databento.com
|
|
34
|
+
Project-URL: Documentation, https://databento.com/docs
|
|
35
35
|
Project-URL: Repository, https://github.com/databento/databento-python
|
|
36
|
+
Project-URL: Bug Tracker, https://github.com/databento/databento-python/issues
|
|
36
37
|
Description-Content-Type: text/markdown
|
|
37
38
|
|
|
38
39
|
# databento-python
|
|
@@ -67,21 +68,47 @@ You can find our full client API reference on the [Historical Reference](https:/
|
|
|
67
68
|
## Requirements
|
|
68
69
|
The library is fully compatible with distributions of Anaconda 2023.x and above.
|
|
69
70
|
The minimum dependencies as found in the `pyproject.toml` are also listed below:
|
|
70
|
-
- python
|
|
71
|
-
- aiohttp
|
|
72
|
-
-
|
|
73
|
-
-
|
|
74
|
-
-
|
|
75
|
-
-
|
|
76
|
-
-
|
|
77
|
-
-
|
|
78
|
-
-
|
|
71
|
+
- python >= 3.10
|
|
72
|
+
- aiohttp >= 3.8.3
|
|
73
|
+
- certifi >= 2026.10.19
|
|
74
|
+
- databento-dbn ~= 0.70.0
|
|
75
|
+
- numpy >= 1.23.5
|
|
76
|
+
- pandas >= 1.5.3
|
|
77
|
+
- pip-system-certs >= 4.0 (Windows only)
|
|
78
|
+
- pyarrow >= 13.0.0
|
|
79
|
+
- requests >= 2.27.0
|
|
80
|
+
- zstandard >= 0.21.0 (Python < 3.14 only)
|
|
79
81
|
|
|
80
82
|
## Installation
|
|
81
83
|
To install the latest stable version of the package from PyPI:
|
|
82
84
|
|
|
83
85
|
pip install -U databento
|
|
84
86
|
|
|
87
|
+
## Building from source
|
|
88
|
+
Building this project requires [uv](https://docs.astral.sh/uv/); no other Python tooling is
|
|
89
|
+
needed, as uv provisions the interpreter and virtual environment for you.
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
git clone https://github.com/databento/databento-python.git
|
|
93
|
+
cd databento-python
|
|
94
|
+
scripts/build.sh
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
This creates a `.venv` in the project directory containing the `dev` dependency group and an
|
|
98
|
+
editable install of the package. From there:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
scripts/test.sh # run the test suite
|
|
102
|
+
scripts/lint.sh # type check with mypy
|
|
103
|
+
uv run python # start a REPL in the environment
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
To produce a wheel and source distribution under `dist/`:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
uv build
|
|
110
|
+
```
|
|
111
|
+
|
|
85
112
|
## Usage
|
|
86
113
|
The library needs to be configured with an API key from your account.
|
|
87
114
|
[Sign up](https://databento.com/signup) for free and you will automatically
|
|
@@ -132,4 +159,3 @@ client = db.Historical()
|
|
|
132
159
|
|
|
133
160
|
## License
|
|
134
161
|
Distributed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html).
|
|
135
|
-
|
|
@@ -30,21 +30,47 @@ You can find our full client API reference on the [Historical Reference](https:/
|
|
|
30
30
|
## Requirements
|
|
31
31
|
The library is fully compatible with distributions of Anaconda 2023.x and above.
|
|
32
32
|
The minimum dependencies as found in the `pyproject.toml` are also listed below:
|
|
33
|
-
- python
|
|
34
|
-
- aiohttp
|
|
35
|
-
-
|
|
36
|
-
-
|
|
37
|
-
-
|
|
38
|
-
-
|
|
39
|
-
-
|
|
40
|
-
-
|
|
41
|
-
-
|
|
33
|
+
- python >= 3.10
|
|
34
|
+
- aiohttp >= 3.8.3
|
|
35
|
+
- certifi >= 2026.10.19
|
|
36
|
+
- databento-dbn ~= 0.70.0
|
|
37
|
+
- numpy >= 1.23.5
|
|
38
|
+
- pandas >= 1.5.3
|
|
39
|
+
- pip-system-certs >= 4.0 (Windows only)
|
|
40
|
+
- pyarrow >= 13.0.0
|
|
41
|
+
- requests >= 2.27.0
|
|
42
|
+
- zstandard >= 0.21.0 (Python < 3.14 only)
|
|
42
43
|
|
|
43
44
|
## Installation
|
|
44
45
|
To install the latest stable version of the package from PyPI:
|
|
45
46
|
|
|
46
47
|
pip install -U databento
|
|
47
48
|
|
|
49
|
+
## Building from source
|
|
50
|
+
Building this project requires [uv](https://docs.astral.sh/uv/); no other Python tooling is
|
|
51
|
+
needed, as uv provisions the interpreter and virtual environment for you.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
git clone https://github.com/databento/databento-python.git
|
|
55
|
+
cd databento-python
|
|
56
|
+
scripts/build.sh
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
This creates a `.venv` in the project directory containing the `dev` dependency group and an
|
|
60
|
+
editable install of the package. From there:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
scripts/test.sh # run the test suite
|
|
64
|
+
scripts/lint.sh # type check with mypy
|
|
65
|
+
uv run python # start a REPL in the environment
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
To produce a wheel and source distribution under `dist/`:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
uv build
|
|
72
|
+
```
|
|
73
|
+
|
|
48
74
|
## Usage
|
|
49
75
|
The library needs to be configured with an API key from your account.
|
|
50
76
|
[Sign up](https://databento.com/signup) for free and you will automatically
|
|
@@ -30,7 +30,6 @@ import numpy as np
|
|
|
30
30
|
import pandas as pd
|
|
31
31
|
import pyarrow as pa
|
|
32
32
|
import pyarrow.parquet as pq
|
|
33
|
-
import zstandard
|
|
34
33
|
from databento_dbn import FIXED_PRICE_SCALE
|
|
35
34
|
from databento_dbn import UNDEF_PRICE
|
|
36
35
|
from databento_dbn import Compression
|
|
@@ -47,6 +46,7 @@ from databento_dbn import SType
|
|
|
47
46
|
from databento_dbn import Transcoder
|
|
48
47
|
from databento_dbn import VersionUpgradePolicy
|
|
49
48
|
|
|
49
|
+
from databento.common import zstd
|
|
50
50
|
from databento.common.constants import DEFINITION_TYPE_MAX_MAP
|
|
51
51
|
from databento.common.constants import SCHEMA_STRUCT_MAP
|
|
52
52
|
from databento.common.constants import SCHEMA_STRUCT_MAP_V1
|
|
@@ -86,12 +86,7 @@ def is_zstandard(reader: IO[bytes]) -> bool:
|
|
|
86
86
|
|
|
87
87
|
"""
|
|
88
88
|
reader.seek(0) # ensure we read from the beginning
|
|
89
|
-
|
|
90
|
-
zstandard.get_frame_parameters(reader.read(18))
|
|
91
|
-
except zstandard.ZstdError:
|
|
92
|
-
return False
|
|
93
|
-
else:
|
|
94
|
-
return True
|
|
89
|
+
return zstd.is_frame_header(reader.read(18))
|
|
95
90
|
|
|
96
91
|
|
|
97
92
|
def is_dbn(reader: IO[bytes]) -> bool:
|
|
@@ -356,7 +351,7 @@ class DBNStore:
|
|
|
356
351
|
|
|
357
352
|
if is_zstandard(buffer):
|
|
358
353
|
self._compression = Compression.ZSTD
|
|
359
|
-
buffer =
|
|
354
|
+
buffer = zstd.stream_reader(data_source.reader)
|
|
360
355
|
elif is_dbn(buffer):
|
|
361
356
|
self._compression = Compression.NONE
|
|
362
357
|
buffer = data_source.reader
|
|
@@ -540,10 +535,7 @@ class DBNStore:
|
|
|
540
535
|
|
|
541
536
|
"""
|
|
542
537
|
if self.compression == Compression.ZSTD:
|
|
543
|
-
return
|
|
544
|
-
self._data_source.reader,
|
|
545
|
-
read_across_frames=True,
|
|
546
|
-
)
|
|
538
|
+
return zstd.stream_reader(self._data_source.reader)
|
|
547
539
|
return self._data_source.reader
|
|
548
540
|
|
|
549
541
|
@property
|
|
@@ -1102,7 +1094,7 @@ class DBNStore:
|
|
|
1102
1094
|
compression = validate_maybe_enum(compression, Compression, "compression")
|
|
1103
1095
|
file_path = validate_file_write_path(path, "path", exist_ok=mode == "w")
|
|
1104
1096
|
|
|
1105
|
-
writer: IO[bytes]
|
|
1097
|
+
writer: IO[bytes]
|
|
1106
1098
|
if compression is None or compression == self.compression:
|
|
1107
1099
|
# Handle trivial case
|
|
1108
1100
|
with open(file_path, mode=f"{mode}b") as writer:
|
|
@@ -1112,12 +1104,7 @@ class DBNStore:
|
|
|
1112
1104
|
return
|
|
1113
1105
|
|
|
1114
1106
|
if compression == Compression.ZSTD:
|
|
1115
|
-
writer =
|
|
1116
|
-
write_checksum=True,
|
|
1117
|
-
).stream_writer(
|
|
1118
|
-
open(file_path, mode=f"{mode}b"),
|
|
1119
|
-
closefd=True,
|
|
1120
|
-
)
|
|
1107
|
+
writer = zstd.stream_writer(file_path, mode)
|
|
1121
1108
|
else:
|
|
1122
1109
|
writer = open(file_path, mode=f"{mode}b")
|
|
1123
1110
|
|
|
@@ -12,9 +12,9 @@ from typing import IO
|
|
|
12
12
|
from typing import Any
|
|
13
13
|
|
|
14
14
|
import pandas as pd
|
|
15
|
-
import zstandard
|
|
16
15
|
from databento_dbn import SType
|
|
17
16
|
|
|
17
|
+
from databento.common import zstd
|
|
18
18
|
from databento.common.constants import ALL_SYMBOLS
|
|
19
19
|
from databento.common.enums import JobState
|
|
20
20
|
from databento.common.validation import validate_enum
|
|
@@ -474,8 +474,7 @@ def convert_jsonl_to_df(data: bytes, compressed: bool) -> pd.DataFrame:
|
|
|
474
474
|
|
|
475
475
|
"""
|
|
476
476
|
if compressed:
|
|
477
|
-
|
|
478
|
-
reader: IO[bytes] = decompressor.stream_reader(data)
|
|
477
|
+
reader: IO[bytes] = zstd.stream_reader(BytesIO(data))
|
|
479
478
|
else:
|
|
480
479
|
reader = BytesIO(data)
|
|
481
480
|
|
|
@@ -70,9 +70,8 @@ def _validate_path_pair(
|
|
|
70
70
|
f"{in_file_valid.stem}_mapped_{i}{in_file_valid.suffix}",
|
|
71
71
|
)
|
|
72
72
|
i += 1
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
raise ValueError("The input file cannot be the same path as the output file.")
|
|
73
|
+
if out_file_valid.exists() and in_file_valid.samefile(out_file_valid):
|
|
74
|
+
raise ValueError("The input file cannot be the same as the output file.")
|
|
76
75
|
|
|
77
76
|
return in_file_valid, out_file_valid
|
|
78
77
|
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from io import BytesIO
|
|
5
|
+
from os import PathLike
|
|
6
|
+
from typing import IO
|
|
7
|
+
from typing import Literal
|
|
8
|
+
from typing import Protocol
|
|
9
|
+
from typing import cast
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class IncrementalCompressor(Protocol):
|
|
13
|
+
"""
|
|
14
|
+
A compressor that accepts uncompressed data in chunks.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def compress(self, data: bytes) -> bytes: ...
|
|
18
|
+
|
|
19
|
+
def flush(self) -> bytes: ...
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
if sys.version_info >= (3, 14):
|
|
23
|
+
from compression import zstd as _zstd
|
|
24
|
+
|
|
25
|
+
ZstdError = _zstd.ZstdError
|
|
26
|
+
|
|
27
|
+
def is_frame_header(header: bytes) -> bool:
|
|
28
|
+
"""
|
|
29
|
+
Determine if `header` is the start of a zstandard frame.
|
|
30
|
+
|
|
31
|
+
Parameters
|
|
32
|
+
----------
|
|
33
|
+
header : bytes
|
|
34
|
+
The first bytes of the data to check, at least 18 bytes to cover
|
|
35
|
+
the largest frame header.
|
|
36
|
+
|
|
37
|
+
Returns
|
|
38
|
+
-------
|
|
39
|
+
bool
|
|
40
|
+
|
|
41
|
+
"""
|
|
42
|
+
try:
|
|
43
|
+
_zstd.get_frame_info(header)
|
|
44
|
+
except _zstd.ZstdError:
|
|
45
|
+
return False
|
|
46
|
+
return True
|
|
47
|
+
|
|
48
|
+
def stream_reader(reader: IO[bytes]) -> IO[bytes]:
|
|
49
|
+
"""
|
|
50
|
+
Return a reader that decompresses zstandard data from `reader`.
|
|
51
|
+
|
|
52
|
+
The returned reader decompresses across all frames in `reader`.
|
|
53
|
+
|
|
54
|
+
Parameters
|
|
55
|
+
----------
|
|
56
|
+
reader : IO[bytes]
|
|
57
|
+
The compressed data to read from.
|
|
58
|
+
|
|
59
|
+
Returns
|
|
60
|
+
-------
|
|
61
|
+
IO[bytes]
|
|
62
|
+
|
|
63
|
+
"""
|
|
64
|
+
return cast(IO[bytes], _zstd.ZstdFile(reader, mode="rb"))
|
|
65
|
+
|
|
66
|
+
def stream_writer(
|
|
67
|
+
path: PathLike[str] | str,
|
|
68
|
+
mode: Literal["w", "x"],
|
|
69
|
+
) -> IO[bytes]:
|
|
70
|
+
"""
|
|
71
|
+
Open the file at `path` for writing zstandard compressed data.
|
|
72
|
+
|
|
73
|
+
Closing the returned writer finishes the frame and closes the file.
|
|
74
|
+
Frames are written with a content checksum.
|
|
75
|
+
|
|
76
|
+
Parameters
|
|
77
|
+
----------
|
|
78
|
+
path : PathLike[str] or str
|
|
79
|
+
The file path to write to.
|
|
80
|
+
mode : str
|
|
81
|
+
The file write mode to use, either "x" or "w".
|
|
82
|
+
|
|
83
|
+
Returns
|
|
84
|
+
-------
|
|
85
|
+
IO[bytes]
|
|
86
|
+
|
|
87
|
+
"""
|
|
88
|
+
return cast(
|
|
89
|
+
IO[bytes],
|
|
90
|
+
_zstd.ZstdFile(
|
|
91
|
+
path,
|
|
92
|
+
mode="wb" if mode == "w" else "xb",
|
|
93
|
+
options={_zstd.CompressionParameter.checksum_flag: 1},
|
|
94
|
+
),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
def compress(data: bytes) -> bytes:
|
|
98
|
+
"""
|
|
99
|
+
Compress `data` into a single zstandard frame.
|
|
100
|
+
|
|
101
|
+
Parameters
|
|
102
|
+
----------
|
|
103
|
+
data : bytes
|
|
104
|
+
The data to compress.
|
|
105
|
+
|
|
106
|
+
Returns
|
|
107
|
+
-------
|
|
108
|
+
bytes
|
|
109
|
+
|
|
110
|
+
"""
|
|
111
|
+
return _zstd.compress(data)
|
|
112
|
+
|
|
113
|
+
def decompress(data: bytes) -> bytes:
|
|
114
|
+
"""
|
|
115
|
+
Decompress all zstandard frames in `data`.
|
|
116
|
+
|
|
117
|
+
Parameters
|
|
118
|
+
----------
|
|
119
|
+
data : bytes
|
|
120
|
+
The compressed data.
|
|
121
|
+
|
|
122
|
+
Returns
|
|
123
|
+
-------
|
|
124
|
+
bytes
|
|
125
|
+
|
|
126
|
+
"""
|
|
127
|
+
return _zstd.decompress(data)
|
|
128
|
+
|
|
129
|
+
def incremental_compressor() -> IncrementalCompressor:
|
|
130
|
+
"""
|
|
131
|
+
Create a compressor that accepts uncompressed data in chunks.
|
|
132
|
+
|
|
133
|
+
Returns
|
|
134
|
+
-------
|
|
135
|
+
IncrementalCompressor
|
|
136
|
+
|
|
137
|
+
"""
|
|
138
|
+
return _zstd.ZstdCompressor()
|
|
139
|
+
|
|
140
|
+
else:
|
|
141
|
+
import zstandard
|
|
142
|
+
|
|
143
|
+
ZstdError = zstandard.ZstdError
|
|
144
|
+
|
|
145
|
+
def is_frame_header(header: bytes) -> bool:
|
|
146
|
+
"""
|
|
147
|
+
Determine if `header` is the start of a zstandard frame.
|
|
148
|
+
|
|
149
|
+
Parameters
|
|
150
|
+
----------
|
|
151
|
+
header : bytes
|
|
152
|
+
The first bytes of the data to check, at least 18 bytes to cover
|
|
153
|
+
the largest frame header.
|
|
154
|
+
|
|
155
|
+
Returns
|
|
156
|
+
-------
|
|
157
|
+
bool
|
|
158
|
+
|
|
159
|
+
"""
|
|
160
|
+
try:
|
|
161
|
+
zstandard.get_frame_parameters(header)
|
|
162
|
+
except zstandard.ZstdError:
|
|
163
|
+
return False
|
|
164
|
+
return True
|
|
165
|
+
|
|
166
|
+
def stream_reader(reader: IO[bytes]) -> IO[bytes]:
|
|
167
|
+
"""
|
|
168
|
+
Return a reader that decompresses zstandard data from `reader`.
|
|
169
|
+
|
|
170
|
+
The returned reader decompresses across all frames in `reader`.
|
|
171
|
+
|
|
172
|
+
Parameters
|
|
173
|
+
----------
|
|
174
|
+
reader : IO[bytes]
|
|
175
|
+
The compressed data to read from.
|
|
176
|
+
|
|
177
|
+
Returns
|
|
178
|
+
-------
|
|
179
|
+
IO[bytes]
|
|
180
|
+
|
|
181
|
+
"""
|
|
182
|
+
return zstandard.ZstdDecompressor().stream_reader(
|
|
183
|
+
reader,
|
|
184
|
+
read_across_frames=True,
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
def stream_writer(
|
|
188
|
+
path: PathLike[str] | str,
|
|
189
|
+
mode: Literal["w", "x"],
|
|
190
|
+
) -> IO[bytes]:
|
|
191
|
+
"""
|
|
192
|
+
Open the file at `path` for writing zstandard compressed data.
|
|
193
|
+
|
|
194
|
+
Closing the returned writer finishes the frame and closes the file.
|
|
195
|
+
Frames are written with a content checksum.
|
|
196
|
+
|
|
197
|
+
Parameters
|
|
198
|
+
----------
|
|
199
|
+
path : PathLike[str] or str
|
|
200
|
+
The file path to write to.
|
|
201
|
+
mode : str
|
|
202
|
+
The file write mode to use, either "x" or "w".
|
|
203
|
+
|
|
204
|
+
Returns
|
|
205
|
+
-------
|
|
206
|
+
IO[bytes]
|
|
207
|
+
|
|
208
|
+
"""
|
|
209
|
+
return zstandard.ZstdCompressor(write_checksum=True).stream_writer(
|
|
210
|
+
open(path, mode=f"{mode}b"),
|
|
211
|
+
closefd=True,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
def compress(data: bytes) -> bytes:
|
|
215
|
+
"""
|
|
216
|
+
Compress `data` into a single zstandard frame.
|
|
217
|
+
|
|
218
|
+
Parameters
|
|
219
|
+
----------
|
|
220
|
+
data : bytes
|
|
221
|
+
The data to compress.
|
|
222
|
+
|
|
223
|
+
Returns
|
|
224
|
+
-------
|
|
225
|
+
bytes
|
|
226
|
+
|
|
227
|
+
"""
|
|
228
|
+
return zstandard.compress(data)
|
|
229
|
+
|
|
230
|
+
def decompress(data: bytes) -> bytes:
|
|
231
|
+
"""
|
|
232
|
+
Decompress all zstandard frames in `data`.
|
|
233
|
+
|
|
234
|
+
Parameters
|
|
235
|
+
----------
|
|
236
|
+
data : bytes
|
|
237
|
+
The compressed data.
|
|
238
|
+
|
|
239
|
+
Returns
|
|
240
|
+
-------
|
|
241
|
+
bytes
|
|
242
|
+
|
|
243
|
+
"""
|
|
244
|
+
return stream_reader(BytesIO(data)).read()
|
|
245
|
+
|
|
246
|
+
def incremental_compressor() -> IncrementalCompressor:
|
|
247
|
+
"""
|
|
248
|
+
Create a compressor that accepts uncompressed data in chunks.
|
|
249
|
+
|
|
250
|
+
Returns
|
|
251
|
+
-------
|
|
252
|
+
IncrementalCompressor
|
|
253
|
+
|
|
254
|
+
"""
|
|
255
|
+
return zstandard.ZstdCompressor().compressobj()
|
|
@@ -223,12 +223,15 @@ class BatchHttpAPI(BentoHttpAPI):
|
|
|
223
223
|
self,
|
|
224
224
|
states: Iterable[JobState | str] | JobState | str | None = "queued,processing,done",
|
|
225
225
|
since: pd.Timestamp | datetime | date | str | int | None = None,
|
|
226
|
-
short: bool | None = None,
|
|
227
226
|
) -> list[dict[str, Any]]:
|
|
228
227
|
"""
|
|
229
|
-
Request all batch
|
|
228
|
+
Request all batch jobs for the user account.
|
|
230
229
|
|
|
231
|
-
|
|
230
|
+
Each job is returned in a list with objects containing the `id`,
|
|
231
|
+
`state`, and `ts_received` fields. Use `get_job_details` to request
|
|
232
|
+
the full details of a job.
|
|
233
|
+
|
|
234
|
+
The jobs will be sorted in order of `ts_received`.
|
|
232
235
|
|
|
233
236
|
Makes a `GET /batch.list_jobs` HTTP request.
|
|
234
237
|
|
|
@@ -239,13 +242,11 @@ class BatchHttpAPI(BentoHttpAPI):
|
|
|
239
242
|
Defaults to all except 'expired'.
|
|
240
243
|
since : pd.Timestamp, datetime, date, str, or int, optional
|
|
241
244
|
The filter for timestamp submitted (will not include jobs prior to this).
|
|
242
|
-
short : bool, optional
|
|
243
|
-
If True, return the short response containing only the `id`, `state`, and `ts_received` fields.
|
|
244
245
|
|
|
245
246
|
Returns
|
|
246
247
|
-------
|
|
247
248
|
list[dict[str, Any]]
|
|
248
|
-
The batch
|
|
249
|
+
The batch jobs.
|
|
249
250
|
|
|
250
251
|
"""
|
|
251
252
|
params: list[tuple[str, str | None]] = [
|
|
@@ -253,11 +254,6 @@ class BatchHttpAPI(BentoHttpAPI):
|
|
|
253
254
|
("since", optional_datetime_to_string(since)),
|
|
254
255
|
]
|
|
255
256
|
|
|
256
|
-
if short is not None:
|
|
257
|
-
params.append(
|
|
258
|
-
("short", str(short)),
|
|
259
|
-
)
|
|
260
|
-
|
|
261
257
|
return self._get(
|
|
262
258
|
url=self._base_url + ".list_jobs",
|
|
263
259
|
params=params,
|
|
@@ -240,7 +240,7 @@ class MetadataHttpAPI(BentoHttpAPI):
|
|
|
240
240
|
def get_dataset_range(
|
|
241
241
|
self,
|
|
242
242
|
dataset: Dataset | str,
|
|
243
|
-
) -> dict[str, str]:
|
|
243
|
+
) -> dict[str, str | dict[str, dict[str, str]]]:
|
|
244
244
|
"""
|
|
245
245
|
Request the available range for the dataset given the user's
|
|
246
246
|
entitlements.
|
|
@@ -254,7 +254,7 @@ class MetadataHttpAPI(BentoHttpAPI):
|
|
|
254
254
|
|
|
255
255
|
Returns
|
|
256
256
|
-------
|
|
257
|
-
dict[str, str | dict[str, str]]
|
|
257
|
+
dict[str, str | dict[str, dict[str, str]]]
|
|
258
258
|
The available range for the dataset.
|
|
259
259
|
|
|
260
260
|
"""
|
|
@@ -34,6 +34,7 @@ from databento.common.types import RecordCallback
|
|
|
34
34
|
from databento.common.validation import validate_enum
|
|
35
35
|
from databento.common.validation import validate_semantic_string
|
|
36
36
|
from databento.live.gateway import SubscriptionRequest
|
|
37
|
+
from databento.live.gateway import UnsubscribeRequest
|
|
37
38
|
from databento.live.session import DEFAULT_REMOTE_PORT
|
|
38
39
|
from databento.live.session import LiveSession
|
|
39
40
|
from databento.live.session import SessionMetadata
|
|
@@ -287,7 +288,32 @@ class Live:
|
|
|
287
288
|
Live.subscribe()
|
|
288
289
|
|
|
289
290
|
"""
|
|
290
|
-
return self._session._subscriptions
|
|
291
|
+
return [requests for _, requests in self._session._subscriptions]
|
|
292
|
+
|
|
293
|
+
@property
|
|
294
|
+
def unsubscription_requests(
|
|
295
|
+
self,
|
|
296
|
+
) -> list[tuple[UnsubscribeRequest, ...]]:
|
|
297
|
+
"""
|
|
298
|
+
Return a list of tuples containing every `UnsubscribeRequest` message
|
|
299
|
+
sent for the session, in the order the unsubscriptions were made.
|
|
300
|
+
|
|
301
|
+
Unsubscriptions which contain a large list of symbols are batched.
|
|
302
|
+
Because of this, a single call to `Live.unsubscribe()` may have more
|
|
303
|
+
than one associated `UnsubscribeRequest`.
|
|
304
|
+
|
|
305
|
+
Returns
|
|
306
|
+
-------
|
|
307
|
+
list[tuple[UnsubscribeRequest, ...]]
|
|
308
|
+
A list of tuples containing every unsubscribe request.
|
|
309
|
+
Each entry in the list corresponds to a single unsubscription.
|
|
310
|
+
|
|
311
|
+
See Also
|
|
312
|
+
--------
|
|
313
|
+
Live.unsubscribe()
|
|
314
|
+
|
|
315
|
+
"""
|
|
316
|
+
return [requests for _, requests in self._session._unsubscriptions]
|
|
291
317
|
|
|
292
318
|
@property
|
|
293
319
|
def symbology_map(self) -> dict[int, str | int]:
|
|
@@ -598,6 +624,61 @@ class Live:
|
|
|
598
624
|
snapshot=snapshot,
|
|
599
625
|
)
|
|
600
626
|
|
|
627
|
+
def unsubscribe(
|
|
628
|
+
self,
|
|
629
|
+
schema: Schema | str,
|
|
630
|
+
symbols: Iterable[str | int] | str | int,
|
|
631
|
+
stype_in: SType | str = SType.RAW_SYMBOL,
|
|
632
|
+
) -> None:
|
|
633
|
+
"""
|
|
634
|
+
Remove the specified symbols from the session's subscriptions for a
|
|
635
|
+
schema.
|
|
636
|
+
|
|
637
|
+
The symbols don't need to match an earlier `Live.subscribe` call: any
|
|
638
|
+
subset of the subscribed symbols can be removed, and `ALL_SYMBOLS`
|
|
639
|
+
removes every symbol for the schema. Symbols that aren't subscribed,
|
|
640
|
+
or that are only subscribed through a parent symbol, are left in place
|
|
641
|
+
and reported in a non-fatal error record.
|
|
642
|
+
|
|
643
|
+
Parameters
|
|
644
|
+
----------
|
|
645
|
+
schema : Schema or str
|
|
646
|
+
The schema to unsubscribe from.
|
|
647
|
+
symbols : Iterable[str | int] or str or int
|
|
648
|
+
The symbols to unsubscribe from.
|
|
649
|
+
stype_in : SType or str, default 'raw_symbol'
|
|
650
|
+
The symbology type of `symbols`. Must be 'raw_symbol', 'parent',
|
|
651
|
+
or 'instrument_id'.
|
|
652
|
+
|
|
653
|
+
Raises
|
|
654
|
+
------
|
|
655
|
+
ValueError
|
|
656
|
+
If called before a connection has been made.
|
|
657
|
+
|
|
658
|
+
See Also
|
|
659
|
+
--------
|
|
660
|
+
Live.subscribe
|
|
661
|
+
Live.start
|
|
662
|
+
|
|
663
|
+
"""
|
|
664
|
+
logger.info(
|
|
665
|
+
"unsubscribing from schema=%s stype_in=%s symbols='%s'",
|
|
666
|
+
schema,
|
|
667
|
+
stype_in,
|
|
668
|
+
symbols,
|
|
669
|
+
)
|
|
670
|
+
if self._session is None:
|
|
671
|
+
raise ValueError("cannot unsubscribe before a connection has been made")
|
|
672
|
+
|
|
673
|
+
schema = validate_enum(schema, Schema, "schema")
|
|
674
|
+
stype_in = validate_enum(stype_in, SType, "stype_in")
|
|
675
|
+
|
|
676
|
+
self._session.unsubscribe(
|
|
677
|
+
schema=schema,
|
|
678
|
+
stype_in=stype_in,
|
|
679
|
+
symbols=symbols,
|
|
680
|
+
)
|
|
681
|
+
|
|
601
682
|
def terminate(self) -> None:
|
|
602
683
|
"""
|
|
603
684
|
Terminate the session and stop processing records immediately.
|
|
@@ -140,6 +140,20 @@ class SubscriptionRequest(GatewayControl):
|
|
|
140
140
|
is_last: int = 1
|
|
141
141
|
|
|
142
142
|
|
|
143
|
+
@dataclasses.dataclass
|
|
144
|
+
class UnsubscribeRequest(GatewayControl):
|
|
145
|
+
"""
|
|
146
|
+
An unsubscribe request is sent to the gateway upon request from the client.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
# The gateway identifies an unsubscribe request by this leading key; its value is ignored.
|
|
150
|
+
unsubscribe: int = dataclasses.field(kw_only=True)
|
|
151
|
+
schema: Schema | str
|
|
152
|
+
stype_in: SType
|
|
153
|
+
symbols: str
|
|
154
|
+
is_last: int = 1
|
|
155
|
+
|
|
156
|
+
|
|
143
157
|
@dataclasses.dataclass
|
|
144
158
|
class SessionStart(GatewayControl):
|
|
145
159
|
"""
|
|
@@ -33,6 +33,7 @@ from databento.live.gateway import GatewayDecoder
|
|
|
33
33
|
from databento.live.gateway import Greeting
|
|
34
34
|
from databento.live.gateway import SessionStart
|
|
35
35
|
from databento.live.gateway import SubscriptionRequest
|
|
36
|
+
from databento.live.gateway import UnsubscribeRequest
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
RECV_BUFFER_SIZE: Final = 64 * 2**10 # 64kb
|
|
@@ -362,6 +363,63 @@ class DatabentoLiveProtocol(asyncio.BufferedProtocol):
|
|
|
362
363
|
self.transport.writelines(map(bytes, subscriptions))
|
|
363
364
|
return subscriptions
|
|
364
365
|
|
|
366
|
+
def unsubscribe(
|
|
367
|
+
self,
|
|
368
|
+
schema: Schema | str,
|
|
369
|
+
symbols: Iterable[str | int] | str | int,
|
|
370
|
+
stype_in: SType | str = SType.RAW_SYMBOL,
|
|
371
|
+
) -> list[UnsubscribeRequest]:
|
|
372
|
+
"""
|
|
373
|
+
Send an UnsubscribeRequest to the gateway.
|
|
374
|
+
|
|
375
|
+
Returns a list of all unsubscribe requests sent to the gateway.
|
|
376
|
+
|
|
377
|
+
Parameters
|
|
378
|
+
----------
|
|
379
|
+
schema : Schema or str
|
|
380
|
+
The schema to unsubscribe from.
|
|
381
|
+
symbols : Iterable[str | int] or str or int
|
|
382
|
+
The symbols to unsubscribe from.
|
|
383
|
+
stype_in : SType or str, default 'raw_symbol'
|
|
384
|
+
The input symbology type that was used to resolve the subscription.
|
|
385
|
+
|
|
386
|
+
Returns
|
|
387
|
+
-------
|
|
388
|
+
list[UnsubscribeRequest]
|
|
389
|
+
|
|
390
|
+
"""
|
|
391
|
+
logger.debug(
|
|
392
|
+
"sending unsubscribe request schema=%s stype_in=%s symbols='%s'",
|
|
393
|
+
schema,
|
|
394
|
+
stype_in,
|
|
395
|
+
symbols,
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
stype_in_valid = validate_enum(stype_in, SType, "stype_in")
|
|
399
|
+
symbols_list = symbols_list_to_list(symbols, stype_in_valid)
|
|
400
|
+
|
|
401
|
+
unsubscriptions: list[UnsubscribeRequest] = []
|
|
402
|
+
chunked_symbols = list(chunk(symbols_list, SYMBOL_LIST_BATCH_SIZE))
|
|
403
|
+
last_chunk_idx = len(chunked_symbols) - 1
|
|
404
|
+
for i, batch in enumerate(chunked_symbols):
|
|
405
|
+
batch_str = ",".join(batch)
|
|
406
|
+
message = UnsubscribeRequest(
|
|
407
|
+
unsubscribe=0,
|
|
408
|
+
schema=validate_enum(schema, Schema, "schema"),
|
|
409
|
+
stype_in=stype_in_valid,
|
|
410
|
+
symbols=batch_str,
|
|
411
|
+
is_last=int(i == last_chunk_idx),
|
|
412
|
+
)
|
|
413
|
+
unsubscriptions.append(message)
|
|
414
|
+
|
|
415
|
+
if len(unsubscriptions) > 1:
|
|
416
|
+
logger.debug(
|
|
417
|
+
"batched unsubscribe into %d requests",
|
|
418
|
+
len(unsubscriptions),
|
|
419
|
+
)
|
|
420
|
+
self.transport.writelines(map(bytes, unsubscriptions))
|
|
421
|
+
return unsubscriptions
|
|
422
|
+
|
|
365
423
|
def start(
|
|
366
424
|
self,
|
|
367
425
|
) -> None:
|
|
@@ -2,11 +2,12 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import dataclasses
|
|
5
|
-
import
|
|
5
|
+
import heapq
|
|
6
6
|
import logging
|
|
7
7
|
import math
|
|
8
8
|
import queue
|
|
9
9
|
import threading
|
|
10
|
+
import time
|
|
10
11
|
from collections.abc import Iterable
|
|
11
12
|
from functools import partial
|
|
12
13
|
from typing import Final
|
|
@@ -28,6 +29,7 @@ from databento.common.types import ClientStream
|
|
|
28
29
|
from databento.common.types import ExceptionCallback
|
|
29
30
|
from databento.common.types import ReconnectCallback
|
|
30
31
|
from databento.live.gateway import SubscriptionRequest
|
|
32
|
+
from databento.live.gateway import UnsubscribeRequest
|
|
31
33
|
from databento.live.protocol import DatabentoLiveProtocol
|
|
32
34
|
|
|
33
35
|
|
|
@@ -454,7 +456,8 @@ class LiveSession:
|
|
|
454
456
|
self._transport: asyncio.Transport | None = None
|
|
455
457
|
self._session_id: str | None = None
|
|
456
458
|
|
|
457
|
-
self._subscriptions: list[tuple[SubscriptionRequest, ...]] = []
|
|
459
|
+
self._subscriptions: list[tuple[int, tuple[SubscriptionRequest, ...]]] = []
|
|
460
|
+
self._unsubscriptions: list[tuple[int, tuple[UnsubscribeRequest, ...]]] = []
|
|
458
461
|
self._reconnect_policy = ReconnectPolicy(reconnect_policy)
|
|
459
462
|
self._reconnect_task: asyncio.Task[None] | None = None
|
|
460
463
|
self._heartbeat_monitor_task: asyncio.Task[None] | None = None
|
|
@@ -626,20 +629,56 @@ class LiveSession:
|
|
|
626
629
|
self._connect(dataset=dataset)
|
|
627
630
|
|
|
628
631
|
subscription_id = len(self._subscriptions)
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
subscription_id=subscription_id,
|
|
638
|
-
),
|
|
632
|
+
subscription = tuple(
|
|
633
|
+
self._protocol.subscribe(
|
|
634
|
+
schema=schema,
|
|
635
|
+
symbols=symbols,
|
|
636
|
+
stype_in=stype_in,
|
|
637
|
+
start=start,
|
|
638
|
+
snapshot=snapshot,
|
|
639
|
+
subscription_id=subscription_id,
|
|
639
640
|
),
|
|
640
641
|
)
|
|
642
|
+
self._subscriptions.append((time.time_ns(), subscription))
|
|
641
643
|
return subscription_id
|
|
642
644
|
|
|
645
|
+
def unsubscribe(
|
|
646
|
+
self,
|
|
647
|
+
schema: Schema | str,
|
|
648
|
+
symbols: Iterable[str | int] | str | int,
|
|
649
|
+
stype_in: SType | str = SType.RAW_SYMBOL,
|
|
650
|
+
) -> None:
|
|
651
|
+
"""
|
|
652
|
+
Send an unsubscribe request on the current connection.
|
|
653
|
+
|
|
654
|
+
Parameters
|
|
655
|
+
----------
|
|
656
|
+
schema : Schema or str
|
|
657
|
+
The schema to unsubscribe from.
|
|
658
|
+
symbols : Iterable[str | int] or str or int
|
|
659
|
+
The symbols to unsubscribe from.
|
|
660
|
+
stype_in : SType or str, default 'raw_symbol'
|
|
661
|
+
The input symbology type that was used to resolve the subscription.
|
|
662
|
+
|
|
663
|
+
Raises
|
|
664
|
+
------
|
|
665
|
+
ValueError
|
|
666
|
+
If there is no active connection to the gateway.
|
|
667
|
+
|
|
668
|
+
"""
|
|
669
|
+
with self._lock:
|
|
670
|
+
if self._protocol is None:
|
|
671
|
+
raise ValueError("session is not connected")
|
|
672
|
+
|
|
673
|
+
unsubscription = tuple(
|
|
674
|
+
self._protocol.unsubscribe(
|
|
675
|
+
schema=schema,
|
|
676
|
+
symbols=symbols,
|
|
677
|
+
stype_in=stype_in,
|
|
678
|
+
),
|
|
679
|
+
)
|
|
680
|
+
self._unsubscriptions.append((time.time_ns(), unsubscription))
|
|
681
|
+
|
|
643
682
|
def terminate(self) -> None:
|
|
644
683
|
with self._lock:
|
|
645
684
|
if self._transport is None:
|
|
@@ -812,7 +851,7 @@ class LiveSession:
|
|
|
812
851
|
elif self._metadata.data is not None:
|
|
813
852
|
gap_start = pd.Timestamp(self._metadata.data.start, tz="UTC")
|
|
814
853
|
else:
|
|
815
|
-
gap_start = pd.Timestamp.
|
|
854
|
+
gap_start = pd.Timestamp.now(tz="UTC")
|
|
816
855
|
|
|
817
856
|
if self._transport is not None:
|
|
818
857
|
self._transport.abort()
|
|
@@ -820,15 +859,7 @@ class LiveSession:
|
|
|
820
859
|
dataset=self._protocol._dataset,
|
|
821
860
|
)
|
|
822
861
|
|
|
823
|
-
|
|
824
|
-
self._protocol.subscribe(
|
|
825
|
-
schema=sub.schema,
|
|
826
|
-
symbols=sub.symbols,
|
|
827
|
-
stype_in=sub.stype_in,
|
|
828
|
-
snapshot=bool(sub.snapshot),
|
|
829
|
-
start=None,
|
|
830
|
-
subscription_id=sub.id,
|
|
831
|
-
)
|
|
862
|
+
self._replay_requests()
|
|
832
863
|
|
|
833
864
|
if should_restart:
|
|
834
865
|
if self._heartbeat_monitor_task is not None:
|
|
@@ -851,6 +882,48 @@ class LiveSession:
|
|
|
851
882
|
else:
|
|
852
883
|
return
|
|
853
884
|
|
|
885
|
+
def _replay_requests(self) -> None:
|
|
886
|
+
"""
|
|
887
|
+
Resend subscriptions and unsubscriptions in the order they were sent so
|
|
888
|
+
the gateway ends up with the same symbols.
|
|
889
|
+
|
|
890
|
+
Subscriptions are resent without their original `start`. Ties keep the
|
|
891
|
+
subscription first.
|
|
892
|
+
|
|
893
|
+
"""
|
|
894
|
+
requests = heapq.merge(
|
|
895
|
+
(
|
|
896
|
+
(sent_at, partial(self._resend_subscriptions, batch))
|
|
897
|
+
for sent_at, batch in self._subscriptions
|
|
898
|
+
),
|
|
899
|
+
(
|
|
900
|
+
(sent_at, partial(self._resend_unsubscriptions, batch))
|
|
901
|
+
for sent_at, batch in self._unsubscriptions
|
|
902
|
+
),
|
|
903
|
+
key=lambda entry: entry[0],
|
|
904
|
+
)
|
|
905
|
+
for _, replay in requests:
|
|
906
|
+
replay()
|
|
907
|
+
|
|
908
|
+
def _resend_subscriptions(self, subscriptions: tuple[SubscriptionRequest, ...]) -> None:
|
|
909
|
+
for sub in subscriptions:
|
|
910
|
+
self._protocol.subscribe(
|
|
911
|
+
schema=sub.schema,
|
|
912
|
+
symbols=sub.symbols,
|
|
913
|
+
stype_in=sub.stype_in,
|
|
914
|
+
snapshot=bool(sub.snapshot),
|
|
915
|
+
start=None,
|
|
916
|
+
subscription_id=sub.id,
|
|
917
|
+
)
|
|
918
|
+
|
|
919
|
+
def _resend_unsubscriptions(self, unsubscriptions: tuple[UnsubscribeRequest, ...]) -> None:
|
|
920
|
+
for unsub in unsubscriptions:
|
|
921
|
+
self._protocol.unsubscribe(
|
|
922
|
+
schema=unsub.schema,
|
|
923
|
+
symbols=unsub.symbols,
|
|
924
|
+
stype_in=unsub.stype_in,
|
|
925
|
+
)
|
|
926
|
+
|
|
854
927
|
def _dispatch_reconnect_callbacks(
|
|
855
928
|
self,
|
|
856
929
|
gap_start: pd.Timestamp,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.89.0"
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "databento"
|
|
3
|
+
version = "0.89.0"
|
|
4
|
+
description = "Official Python client library for Databento"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
license = "Apache-2.0"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 4 - Beta",
|
|
11
|
+
"Operating System :: OS Independent",
|
|
12
|
+
"Programming Language :: Python :: 3",
|
|
13
|
+
"Programming Language :: Python :: 3.10",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Programming Language :: Python :: 3.14",
|
|
18
|
+
"Topic :: Office/Business :: Financial",
|
|
19
|
+
"Topic :: Office/Business :: Financial :: Investment",
|
|
20
|
+
"Topic :: Software Development :: Libraries",
|
|
21
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"aiohttp>=3.8.3,<4.0.0; python_version < '3.12'",
|
|
25
|
+
"aiohttp>=3.9.0,<4.0.0; python_version >= '3.12'",
|
|
26
|
+
"certifi>=2026.10.19",
|
|
27
|
+
"databento-dbn~=0.70.0",
|
|
28
|
+
"numpy>=1.23.5; python_version < '3.12'",
|
|
29
|
+
"numpy>=1.26.0; python_version >= '3.12'",
|
|
30
|
+
"pandas>=1.5.3,<4.0.0",
|
|
31
|
+
"pip-system-certs>=4.0; platform_system == 'Windows'",
|
|
32
|
+
"pyarrow>=13.0.0",
|
|
33
|
+
"requests>=2.27.0",
|
|
34
|
+
"zstandard>=0.21.0; python_version < '3.14'",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[[project.authors]]
|
|
38
|
+
name = "Databento"
|
|
39
|
+
email = "support@databento.com"
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://databento.com"
|
|
43
|
+
Documentation = "https://databento.com/docs"
|
|
44
|
+
Repository = "https://github.com/databento/databento-python"
|
|
45
|
+
"Bug Tracker" = "https://github.com/databento/databento-python/issues"
|
|
46
|
+
|
|
47
|
+
[dependency-groups]
|
|
48
|
+
dev = [
|
|
49
|
+
"black>=25.9.0,<26",
|
|
50
|
+
"mypy>=1.19.0,<2",
|
|
51
|
+
"pytest>=8.4.2,<9",
|
|
52
|
+
"pytest-asyncio>=1.2.0,<2",
|
|
53
|
+
"ruff>=0.15.12,<0.16",
|
|
54
|
+
"teamcity-messages>=1.32,<2",
|
|
55
|
+
"tomli>=2.0.1,<3",
|
|
56
|
+
"types-requests>=2.30.0,<3",
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
[build-system]
|
|
60
|
+
requires = ["uv_build>=0.12.2,<0.13.0"]
|
|
61
|
+
build-backend = "uv_build"
|
|
62
|
+
|
|
63
|
+
[tool.uv.build-backend]
|
|
64
|
+
module-name = "databento"
|
|
65
|
+
module-root = ""
|
|
66
|
+
|
|
67
|
+
[tool.black]
|
|
68
|
+
line_length = 100
|
|
69
|
+
|
|
70
|
+
[tool.mypy]
|
|
71
|
+
python_version = "3.10"
|
|
72
|
+
disallow_untyped_defs = true
|
|
73
|
+
disallow_any_generics = true
|
|
74
|
+
disallow_subclassing_any = true
|
|
75
|
+
disable_error_code = ["import-untyped"]
|
|
76
|
+
ignore_missing_imports = true
|
|
77
|
+
namespace_packages = true
|
|
78
|
+
no_strict_optional = true
|
|
79
|
+
warn_no_return = true
|
|
80
|
+
warn_unused_configs = true
|
|
81
|
+
warn_unused_ignores = false
|
|
82
|
+
|
|
83
|
+
[tool.pytest.ini_options]
|
|
84
|
+
testpaths = ["tests"]
|
|
85
|
+
asyncio_mode = "auto"
|
|
86
|
+
asyncio_default_fixture_loop_scope = "module"
|
|
87
|
+
asyncio_default_test_loop_scope = "module"
|
|
88
|
+
|
|
89
|
+
[tool.ruff]
|
|
90
|
+
extend = "../ruff.toml"
|
|
@@ -1,15 +1,30 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "databento"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.89.0"
|
|
4
4
|
description = "Official Python client library for Databento"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
7
7
|
license = "Apache-2.0"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
8
9
|
authors = [{ name = "Databento", email = "support@databento.com" }]
|
|
9
|
-
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 4 - Beta",
|
|
12
|
+
"Operating System :: OS Independent",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.10",
|
|
15
|
+
"Programming Language :: Python :: 3.11",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Programming Language :: Python :: 3.13",
|
|
18
|
+
"Programming Language :: Python :: 3.14",
|
|
19
|
+
"Topic :: Office/Business :: Financial",
|
|
20
|
+
"Topic :: Office/Business :: Financial :: Investment",
|
|
21
|
+
"Topic :: Software Development :: Libraries",
|
|
22
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
23
|
+
]
|
|
10
24
|
dependencies = [
|
|
11
25
|
"aiohttp>=3.8.3,<4.0.0; python_version < '3.12'",
|
|
12
26
|
"aiohttp>=3.9.0,<4.0.0; python_version >= '3.12'",
|
|
27
|
+
"certifi>=2026.10.19",
|
|
13
28
|
"databento-dbn~=0.70.0",
|
|
14
29
|
"numpy>=1.23.5; python_version < '3.12'",
|
|
15
30
|
"numpy>=1.26.0; python_version >= '3.12'",
|
|
@@ -17,7 +32,7 @@ dependencies = [
|
|
|
17
32
|
"pip-system-certs>=4.0; platform_system == 'Windows'",
|
|
18
33
|
"pyarrow>=13.0.0",
|
|
19
34
|
"requests>=2.27.0",
|
|
20
|
-
"zstandard>=0.21.0",
|
|
35
|
+
"zstandard>=0.21.0; python_version < '3.14'",
|
|
21
36
|
]
|
|
22
37
|
|
|
23
38
|
[project.urls]
|
|
@@ -26,37 +41,25 @@ Documentation = "https://databento.com/docs"
|
|
|
26
41
|
Repository = "https://github.com/databento/databento-python"
|
|
27
42
|
"Bug Tracker" = "https://github.com/databento/databento-python/issues"
|
|
28
43
|
|
|
29
|
-
[
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
44
|
+
[dependency-groups]
|
|
45
|
+
dev = [
|
|
46
|
+
"black>=25.9.0,<26",
|
|
47
|
+
"mypy>=1.19.0,<2",
|
|
48
|
+
"pytest>=8.4.2,<9",
|
|
49
|
+
"pytest-asyncio>=1.2.0,<2",
|
|
50
|
+
"ruff>=0.15.12,<0.16",
|
|
51
|
+
"teamcity-messages>=1.32,<2",
|
|
52
|
+
"tomli>=2.0.1,<3",
|
|
53
|
+
"types-requests>=2.30.0,<3",
|
|
34
54
|
]
|
|
35
|
-
classifiers = [
|
|
36
|
-
"Development Status :: 4 - Beta",
|
|
37
|
-
"Operating System :: OS Independent",
|
|
38
|
-
"Topic :: Software Development :: Libraries",
|
|
39
|
-
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
40
|
-
"Topic :: Office/Business :: Financial",
|
|
41
|
-
"Topic :: Office/Business :: Financial :: Investment",
|
|
42
|
-
]
|
|
43
|
-
|
|
44
|
-
[tool.poetry.dependencies]
|
|
45
|
-
python = ">=3.10,<3.15"
|
|
46
|
-
|
|
47
|
-
[tool.poetry.group.dev.dependencies]
|
|
48
|
-
black = "^25.9.0"
|
|
49
|
-
mypy = "1.5.1"
|
|
50
|
-
pytest = "^7.4.2"
|
|
51
|
-
pytest-asyncio = "==0.21.1"
|
|
52
|
-
ruff = "^0.15.2"
|
|
53
|
-
tomli = "^2.0.1"
|
|
54
|
-
types-requests = "^2.30.0.0"
|
|
55
|
-
teamcity-messages = "^1.32"
|
|
56
55
|
|
|
57
56
|
[build-system]
|
|
58
|
-
requires = ["
|
|
59
|
-
build-backend = "
|
|
57
|
+
requires = ["uv_build>=0.12.2,<0.13.0"]
|
|
58
|
+
build-backend = "uv_build"
|
|
59
|
+
|
|
60
|
+
[tool.uv.build-backend]
|
|
61
|
+
module-name = "databento"
|
|
62
|
+
module-root = ""
|
|
60
63
|
|
|
61
64
|
[tool.black]
|
|
62
65
|
line_length = 100
|
|
@@ -66,6 +69,7 @@ python_version = "3.10"
|
|
|
66
69
|
disallow_untyped_defs = true
|
|
67
70
|
disallow_any_generics = true
|
|
68
71
|
disallow_subclassing_any = true
|
|
72
|
+
disable_error_code = ["import-untyped"]
|
|
69
73
|
ignore_missing_imports = true
|
|
70
74
|
namespace_packages = true
|
|
71
75
|
no_strict_optional = true
|
|
@@ -76,6 +80,8 @@ warn_unused_ignores = false
|
|
|
76
80
|
[tool.pytest.ini_options]
|
|
77
81
|
testpaths = ["tests"]
|
|
78
82
|
asyncio_mode = "auto"
|
|
83
|
+
asyncio_default_fixture_loop_scope = "module"
|
|
84
|
+
asyncio_default_test_loop_scope = "module"
|
|
79
85
|
|
|
80
86
|
[tool.ruff]
|
|
81
87
|
extend = "../ruff.toml"
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.87.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|