itchfeed 1.0.5__tar.gz → 1.0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {itchfeed-1.0.5/itchfeed.egg-info → itchfeed-1.0.6}/PKG-INFO +16 -38
- {itchfeed-1.0.5 → itchfeed-1.0.6}/README.md +10 -18
- itchfeed-1.0.6/itch/__init__.py +17 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/itch/messages.py +2 -4
- {itchfeed-1.0.5 → itchfeed-1.0.6}/itch/parser.py +105 -69
- {itchfeed-1.0.5 → itchfeed-1.0.6/itchfeed.egg-info}/PKG-INFO +16 -38
- {itchfeed-1.0.5 → itchfeed-1.0.6}/itchfeed.egg-info/SOURCES.txt +3 -1
- itchfeed-1.0.6/pyproject.toml +53 -0
- itchfeed-1.0.6/setup.py +5 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/tests/test_create_message.py +1 -1
- itchfeed-1.0.6/tests/test_parser.py +110 -0
- itchfeed-1.0.5/itch/__init__.py +0 -11
- itchfeed-1.0.5/setup.py +0 -68
- {itchfeed-1.0.5 → itchfeed-1.0.6}/LICENSE +0 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/MANIFEST.in +0 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/itch/indicators.py +0 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/itchfeed.egg-info/dependency_links.txt +0 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/itchfeed.egg-info/requires.txt +0 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/itchfeed.egg-info/top_level.txt +0 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/requirements.txt +0 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/setup.cfg +0 -0
- {itchfeed-1.0.5 → itchfeed-1.0.6}/tests/test_messages.py +0 -0
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: itchfeed
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.6
|
|
4
4
|
Summary: Simple parser for ITCH messages
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
License: The MIT License (MIT)
|
|
5
|
+
Author-email: Bertin Balouki SIMYELI <bertin@bbs-trading.com>
|
|
6
|
+
Maintainer-email: Bertin Balouki SIMYELI <bertin@bbs-trading.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/bbalouki/itch
|
|
9
|
+
Project-URL: Download, https://pypi.org/project/itchfeed/
|
|
11
10
|
Project-URL: Source Code, https://github.com/bbalouki/itch
|
|
12
11
|
Keywords: Finance,Financial,Quantitative,Equities,Totalview-ITCH,Totalview,Nasdaq-ITCH,Nasdaq,ITCH,Data,Feed,ETFs,Funds,Trading,Investing
|
|
13
12
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -19,20 +18,7 @@ Classifier: Operating System :: OS Independent
|
|
|
19
18
|
Description-Content-Type: text/markdown
|
|
20
19
|
License-File: LICENSE
|
|
21
20
|
Requires-Dist: pytest
|
|
22
|
-
Dynamic: author
|
|
23
|
-
Dynamic: author-email
|
|
24
|
-
Dynamic: classifier
|
|
25
|
-
Dynamic: description
|
|
26
|
-
Dynamic: description-content-type
|
|
27
|
-
Dynamic: download-url
|
|
28
|
-
Dynamic: home-page
|
|
29
|
-
Dynamic: keywords
|
|
30
|
-
Dynamic: license
|
|
31
21
|
Dynamic: license-file
|
|
32
|
-
Dynamic: maintainer
|
|
33
|
-
Dynamic: project-url
|
|
34
|
-
Dynamic: requires-dist
|
|
35
|
-
Dynamic: summary
|
|
36
22
|
|
|
37
23
|
# Nasdaq TotalView-ITCH 5.0 Parser
|
|
38
24
|
[](https://pypi.org/project/itchfeed/)
|
|
@@ -111,6 +97,8 @@ After installation (typically via pip), import the necessary modules directly in
|
|
|
111
97
|
|
|
112
98
|
## Usage
|
|
113
99
|
|
|
100
|
+
Download some sample data [here](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/)
|
|
101
|
+
|
|
114
102
|
### Parsing from a Binary File
|
|
115
103
|
|
|
116
104
|
This is useful for processing historical ITCH data stored in files. The `MessageParser` handles buffering efficiently.
|
|
@@ -131,9 +119,8 @@ parser = MessageParser() # Parses all messages by default
|
|
|
131
119
|
|
|
132
120
|
# Path to your ITCH 5.0 data file
|
|
133
121
|
itch_file_path = 'path/to/your/data'
|
|
134
|
-
# you can find sample data [here](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/)
|
|
135
122
|
|
|
136
|
-
# The `
|
|
123
|
+
# The `parse_file()` method reads the ITCH data in chunks.
|
|
137
124
|
# - `cachesize` (optional, default: 65536 bytes): This parameter determines the size of data chunks
|
|
138
125
|
# read from the file at a time. Adjusting this might impact performance for very large files
|
|
139
126
|
# or memory usage, but the default is generally suitable.
|
|
@@ -144,13 +131,9 @@ itch_file_path = 'path/to/your/data'
|
|
|
144
131
|
|
|
145
132
|
try:
|
|
146
133
|
with open(itch_file_path, 'rb') as itch_file:
|
|
147
|
-
#
|
|
148
|
-
parsed_messages = parser.read_message_from_file(itch_file) # You can also pass cachesize here, e.g., parser.read_message_from_file(itch_file, cachesize=131072)
|
|
149
|
-
|
|
150
|
-
print(f"Parsed {len(parsed_messages)} messages.")
|
|
151
|
-
|
|
134
|
+
# parse_file returns an Iterator of parsed message objects
|
|
152
135
|
# Process the messages
|
|
153
|
-
for message in
|
|
136
|
+
for message in parser.parse_file(itch_file):
|
|
154
137
|
# Access attributes directly
|
|
155
138
|
print(f"Type: {message.message_type.decode()}, Timestamp: {message.timestamp}")
|
|
156
139
|
|
|
@@ -192,14 +175,8 @@ parser = MessageParser()
|
|
|
192
175
|
# Example: \x00\x0bS...\x00\x25R...\x00\x27F...
|
|
193
176
|
raw_binary_data: bytes = b"..." # Your raw ITCH 5.0 data chunk
|
|
194
177
|
|
|
195
|
-
#
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
print(f"Parsed {message_queue.qsize()} messages from the byte chunk.")
|
|
199
|
-
|
|
200
|
-
# Process messages from the queue
|
|
201
|
-
while not message_queue.empty():
|
|
202
|
-
message = message_queue.get()
|
|
178
|
+
# parse_stream returns an Iterator of parsed message objects
|
|
179
|
+
for message in parser.parse_stream(raw_binary_data)
|
|
203
180
|
|
|
204
181
|
print(f"Type: {message.message_type.decode()}, Timestamp: {message.timestamp}")
|
|
205
182
|
|
|
@@ -462,11 +439,12 @@ Common scenarios that can lead to a `ValueError` include:
|
|
|
462
439
|
|
|
463
440
|
It's crucial to anticipate these errors in your application:
|
|
464
441
|
|
|
465
|
-
* **Use `try-except` Blocks:** Wrap your parsing calls (especially `
|
|
442
|
+
* **Use `try-except` Blocks:** Wrap your parsing calls (especially `parse_file` or `parse_stream`) in `try-except ValueError as e:` blocks.
|
|
466
443
|
```python
|
|
467
444
|
try:
|
|
468
445
|
# ... parsing operations ...
|
|
469
|
-
|
|
446
|
+
for message in parser.parse_file(itch_file):
|
|
447
|
+
...
|
|
470
448
|
except ValueError as e:
|
|
471
449
|
print(f"An error occurred during parsing: {e}")
|
|
472
450
|
# Log the error, problematic data chunk, or take other actions
|
|
@@ -75,6 +75,8 @@ After installation (typically via pip), import the necessary modules directly in
|
|
|
75
75
|
|
|
76
76
|
## Usage
|
|
77
77
|
|
|
78
|
+
Download some sample data [here](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/)
|
|
79
|
+
|
|
78
80
|
### Parsing from a Binary File
|
|
79
81
|
|
|
80
82
|
This is useful for processing historical ITCH data stored in files. The `MessageParser` handles buffering efficiently.
|
|
@@ -95,9 +97,8 @@ parser = MessageParser() # Parses all messages by default
|
|
|
95
97
|
|
|
96
98
|
# Path to your ITCH 5.0 data file
|
|
97
99
|
itch_file_path = 'path/to/your/data'
|
|
98
|
-
# you can find sample data [here](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/)
|
|
99
100
|
|
|
100
|
-
# The `
|
|
101
|
+
# The `parse_file()` method reads the ITCH data in chunks.
|
|
101
102
|
# - `cachesize` (optional, default: 65536 bytes): This parameter determines the size of data chunks
|
|
102
103
|
# read from the file at a time. Adjusting this might impact performance for very large files
|
|
103
104
|
# or memory usage, but the default is generally suitable.
|
|
@@ -108,13 +109,9 @@ itch_file_path = 'path/to/your/data'
|
|
|
108
109
|
|
|
109
110
|
try:
|
|
110
111
|
with open(itch_file_path, 'rb') as itch_file:
|
|
111
|
-
#
|
|
112
|
-
parsed_messages = parser.read_message_from_file(itch_file) # You can also pass cachesize here, e.g., parser.read_message_from_file(itch_file, cachesize=131072)
|
|
113
|
-
|
|
114
|
-
print(f"Parsed {len(parsed_messages)} messages.")
|
|
115
|
-
|
|
112
|
+
# parse_file returns an Iterator of parsed message objects
|
|
116
113
|
# Process the messages
|
|
117
|
-
for message in
|
|
114
|
+
for message in parser.parse_file(itch_file):
|
|
118
115
|
# Access attributes directly
|
|
119
116
|
print(f"Type: {message.message_type.decode()}, Timestamp: {message.timestamp}")
|
|
120
117
|
|
|
@@ -156,14 +153,8 @@ parser = MessageParser()
|
|
|
156
153
|
# Example: \x00\x0bS...\x00\x25R...\x00\x27F...
|
|
157
154
|
raw_binary_data: bytes = b"..." # Your raw ITCH 5.0 data chunk
|
|
158
155
|
|
|
159
|
-
#
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
print(f"Parsed {message_queue.qsize()} messages from the byte chunk.")
|
|
163
|
-
|
|
164
|
-
# Process messages from the queue
|
|
165
|
-
while not message_queue.empty():
|
|
166
|
-
message = message_queue.get()
|
|
156
|
+
# parse_stream returns an Iterator of parsed message objects
|
|
157
|
+
for message in parser.parse_stream(raw_binary_data)
|
|
167
158
|
|
|
168
159
|
print(f"Type: {message.message_type.decode()}, Timestamp: {message.timestamp}")
|
|
169
160
|
|
|
@@ -426,11 +417,12 @@ Common scenarios that can lead to a `ValueError` include:
|
|
|
426
417
|
|
|
427
418
|
It's crucial to anticipate these errors in your application:
|
|
428
419
|
|
|
429
|
-
* **Use `try-except` Blocks:** Wrap your parsing calls (especially `
|
|
420
|
+
* **Use `try-except` Blocks:** Wrap your parsing calls (especially `parse_file` or `parse_stream`) in `try-except ValueError as e:` blocks.
|
|
430
421
|
```python
|
|
431
422
|
try:
|
|
432
423
|
# ... parsing operations ...
|
|
433
|
-
|
|
424
|
+
for message in parser.parse_file(itch_file):
|
|
425
|
+
...
|
|
434
426
|
except ValueError as e:
|
|
435
427
|
print(f"An error occurred during parsing: {e}")
|
|
436
428
|
# Log the error, problematic data chunk, or take other actions
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Nasdaq TotalView-ITCH 5.0 Parser
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
__author__ = "Bertin Balouki SIMYELI"
|
|
6
|
+
__copyright__ = "2025 Bertin Balouki SIMYELI"
|
|
7
|
+
__email__ = "bertin@bbs-trading.com"
|
|
8
|
+
__license__ = "MIT"
|
|
9
|
+
|
|
10
|
+
from importlib.metadata import version, PackageNotFoundError
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
__version__ = version("itchfeed")
|
|
14
|
+
except PackageNotFoundError:
|
|
15
|
+
__version__ = "unknown"
|
|
16
|
+
|
|
17
|
+
|
|
@@ -46,7 +46,7 @@ class MarketMessage(object):
|
|
|
46
46
|
def __bytes__(self) -> bytes:
|
|
47
47
|
return self.to_bytes()
|
|
48
48
|
|
|
49
|
-
def to_bytes(self) -> bytes:
|
|
49
|
+
def to_bytes(self) -> bytes: # type: ignore
|
|
50
50
|
"""
|
|
51
51
|
Packs the message into bytes using the defined message_pack_format.
|
|
52
52
|
This method should be overridden by subclasses to include specific fields.
|
|
@@ -109,9 +109,7 @@ class MarketMessage(object):
|
|
|
109
109
|
ts1 = self.timestamp >> 32
|
|
110
110
|
ts2 = self.timestamp - (ts1 << 32)
|
|
111
111
|
return (ts1, ts2)
|
|
112
|
-
|
|
113
|
-
ts2 = self.timestamp - (ts1 << 32)
|
|
114
|
-
return (ts1, ts2)
|
|
112
|
+
|
|
115
113
|
|
|
116
114
|
def decode_price(self, price_attr: str) -> float:
|
|
117
115
|
precision = getattr(self, "price_precision")
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import IO, BinaryIO, Iterator
|
|
1
|
+
from typing import IO, BinaryIO, Callable, Iterator, Optional, Tuple
|
|
2
2
|
|
|
3
3
|
from itch.messages import MESSAGES, MarketMessage
|
|
4
4
|
from itch.messages import messages as msgs
|
|
@@ -13,8 +13,70 @@ class MessageParser(object):
|
|
|
13
13
|
def __init__(self, message_type: bytes = MESSAGES):
|
|
14
14
|
self.message_type = message_type
|
|
15
15
|
|
|
16
|
-
def
|
|
17
|
-
|
|
16
|
+
def get_message_type(self, message: bytes) -> MarketMessage:
|
|
17
|
+
"""
|
|
18
|
+
Take an entire bytearray and return the appropriate ITCH message
|
|
19
|
+
instance based on the message type indicator (first byte of the message).
|
|
20
|
+
|
|
21
|
+
All message type indicators are single ASCII characters.
|
|
22
|
+
"""
|
|
23
|
+
message_type = message[0:1]
|
|
24
|
+
try:
|
|
25
|
+
return msgs[message_type](message) # type: ignore
|
|
26
|
+
except Exception:
|
|
27
|
+
raise ValueError(
|
|
28
|
+
f"Unknown message type: {message_type.decode(encoding='ascii')}"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
def _parse_message_from_buffer(
|
|
32
|
+
self, buffer: memoryview, offset: int
|
|
33
|
+
) -> Optional[Tuple[MarketMessage, int]]:
|
|
34
|
+
"""
|
|
35
|
+
Parses a single ITCH message from a memory buffer.
|
|
36
|
+
|
|
37
|
+
This method checks for a 2-byte header (a null byte and a length byte),
|
|
38
|
+
determines the full message size, and extracts the message if the
|
|
39
|
+
complete message is present in the buffer.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
buffer (memoryview):
|
|
43
|
+
The buffer containing the binary data.
|
|
44
|
+
offset (int):
|
|
45
|
+
The starting position in the buffer to begin parsing.
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Optional[Tuple[MarketMessage, int]]:
|
|
49
|
+
A tuple containing the parsed MarketMessage and the total length
|
|
50
|
+
of the message including the header. Returns None if a complete
|
|
51
|
+
message could not be parsed.
|
|
52
|
+
|
|
53
|
+
Raises:
|
|
54
|
+
ValueError:
|
|
55
|
+
If the data at the current offset does not start with the
|
|
56
|
+
expected 0x00 byte.
|
|
57
|
+
"""
|
|
58
|
+
buffer_len = len(buffer)
|
|
59
|
+
if offset + 2 > buffer_len:
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
if buffer[offset : offset + 1] != b"\x00":
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"Unexpected start byte at offset {offset}: "
|
|
65
|
+
f"{buffer[offset : offset + 1].tobytes()}"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
msg_len = buffer[offset + 1]
|
|
69
|
+
total_len = 2 + msg_len
|
|
70
|
+
|
|
71
|
+
if offset + total_len > buffer_len:
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
raw_msg = buffer[offset + 2 : offset + total_len]
|
|
75
|
+
message = self.get_message_type(raw_msg.tobytes())
|
|
76
|
+
return message, total_len
|
|
77
|
+
|
|
78
|
+
def parse_file(
|
|
79
|
+
self, file: BinaryIO, cachesize: int = 65_536, save_file: Optional[IO] = None
|
|
18
80
|
) -> Iterator[MarketMessage]:
|
|
19
81
|
"""
|
|
20
82
|
Reads and parses market messages from a binary file-like object.
|
|
@@ -55,7 +117,7 @@ class MessageParser(object):
|
|
|
55
117
|
>>> with gzip.open(data_file, "rb") as itch_file:
|
|
56
118
|
>>> message_count = 0
|
|
57
119
|
>>> start_time = time.time()
|
|
58
|
-
>>> for message in parser.
|
|
120
|
+
>>> for message in parser.parse_file(itch_file):
|
|
59
121
|
>>> message_count += 1
|
|
60
122
|
>>> if message_count <= 5:
|
|
61
123
|
>>> print(message)
|
|
@@ -66,46 +128,24 @@ class MessageParser(object):
|
|
|
66
128
|
if not file.readable():
|
|
67
129
|
raise ValueError("file must be opened in binary read mode")
|
|
68
130
|
|
|
69
|
-
if save_file is not None:
|
|
70
|
-
|
|
71
|
-
raise ValueError("save_file must be opened in binary write mode")
|
|
131
|
+
if save_file is not None and not save_file.writable():
|
|
132
|
+
raise ValueError("save_file must be opened in binary write mode")
|
|
72
133
|
|
|
73
134
|
data_buffer = b""
|
|
74
135
|
offset = 0
|
|
75
136
|
|
|
76
137
|
while True:
|
|
77
|
-
|
|
138
|
+
parsed = self._parse_message_from_buffer(memoryview(data_buffer), offset)
|
|
139
|
+
if parsed is None:
|
|
78
140
|
data_buffer = data_buffer[offset:]
|
|
79
141
|
offset = 0
|
|
80
|
-
new_data = file.read(cachesize)
|
|
81
|
-
if not new_data:
|
|
82
|
-
break
|
|
83
|
-
data_buffer += new_data
|
|
84
|
-
|
|
85
|
-
if len(data_buffer) < 2:
|
|
86
|
-
break
|
|
87
|
-
|
|
88
|
-
if data_buffer[offset : offset + 1] != b"\x00":
|
|
89
|
-
raise ValueError(
|
|
90
|
-
"Unexpected byte: "
|
|
91
|
-
+ str(data_buffer[offset : offset + 1], encoding="ascii")
|
|
92
|
-
)
|
|
93
|
-
|
|
94
|
-
message_len = data_buffer[offset + 1]
|
|
95
|
-
total_len = 2 + message_len
|
|
96
|
-
|
|
97
|
-
if len(data_buffer) - offset < total_len:
|
|
98
|
-
data_buffer = data_buffer[offset:]
|
|
99
|
-
offset = 0
|
|
100
|
-
|
|
101
142
|
new_data = file.read(cachesize)
|
|
102
143
|
if not new_data:
|
|
103
144
|
break
|
|
104
145
|
data_buffer += new_data
|
|
105
146
|
continue
|
|
106
147
|
|
|
107
|
-
|
|
108
|
-
message = self.get_message_type(message_data)
|
|
148
|
+
message, total_len = parsed
|
|
109
149
|
|
|
110
150
|
if message.message_type in self.message_type:
|
|
111
151
|
if save_file is not None:
|
|
@@ -113,13 +153,16 @@ class MessageParser(object):
|
|
|
113
153
|
save_file.write(b"\x00" + msg_len_to_bytes + message.to_bytes())
|
|
114
154
|
yield message
|
|
115
155
|
|
|
116
|
-
if
|
|
117
|
-
|
|
118
|
-
|
|
156
|
+
if (
|
|
157
|
+
message.message_type == b"S"
|
|
158
|
+
and getattr(message, "event_code", b"") == b"C"
|
|
159
|
+
):
|
|
160
|
+
break
|
|
161
|
+
|
|
119
162
|
offset += total_len
|
|
120
163
|
|
|
121
|
-
def
|
|
122
|
-
self, data: bytes, save_file: IO = None
|
|
164
|
+
def parse_stream(
|
|
165
|
+
self, data: bytes, save_file: Optional[IO] = None
|
|
123
166
|
) -> Iterator[MarketMessage]:
|
|
124
167
|
"""
|
|
125
168
|
Process one or multiple ITCH binary messages from a raw bytes input.
|
|
@@ -139,30 +182,20 @@ class MessageParser(object):
|
|
|
139
182
|
- No buffering is done here — this is meant for real-time decoding.
|
|
140
183
|
"""
|
|
141
184
|
if not isinstance(data, (bytes, bytearray)):
|
|
142
|
-
raise TypeError("data must be bytes or bytearray not " + str(type(data)))
|
|
185
|
+
raise TypeError("data must be bytes or bytearray, not " + str(type(data)))
|
|
143
186
|
|
|
144
|
-
if save_file is not None:
|
|
145
|
-
|
|
146
|
-
raise ValueError("save_file must be opened in binary write mode")
|
|
187
|
+
if save_file is not None and not save_file.writable():
|
|
188
|
+
raise ValueError("save_file must be opened in binary write mode")
|
|
147
189
|
|
|
148
190
|
offset = 0
|
|
149
191
|
data_view = memoryview(data)
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if
|
|
154
|
-
raise ValueError(
|
|
155
|
-
f"Unexpected start byte at offset {offset:offset+1}: "
|
|
156
|
-
f"{data_view[offset : offset + 1].tobytes()}"
|
|
157
|
-
)
|
|
158
|
-
msg_len = data_view[offset + 1]
|
|
159
|
-
total_len = 2 + msg_len
|
|
160
|
-
|
|
161
|
-
if offset + total_len > data_len:
|
|
192
|
+
|
|
193
|
+
while True:
|
|
194
|
+
parsed = self._parse_message_from_buffer(data_view, offset)
|
|
195
|
+
if parsed is None:
|
|
162
196
|
break
|
|
163
197
|
|
|
164
|
-
|
|
165
|
-
message = self.get_message_type(raw_msg.tobytes())
|
|
198
|
+
message, total_len = parsed
|
|
166
199
|
|
|
167
200
|
if message.message_type in self.message_type:
|
|
168
201
|
if save_file is not None:
|
|
@@ -170,23 +203,26 @@ class MessageParser(object):
|
|
|
170
203
|
save_file.write(b"\x00" + msg_len_to_bytes + message.to_bytes())
|
|
171
204
|
yield message
|
|
172
205
|
|
|
173
|
-
if
|
|
174
|
-
|
|
175
|
-
|
|
206
|
+
if (
|
|
207
|
+
message.message_type == b"S"
|
|
208
|
+
and getattr(message, "event_code", b"") == b"C"
|
|
209
|
+
):
|
|
210
|
+
break
|
|
176
211
|
|
|
177
212
|
offset += total_len
|
|
178
213
|
|
|
179
|
-
def
|
|
214
|
+
def parse_messages(
|
|
215
|
+
self,
|
|
216
|
+
data: BinaryIO | bytes | bytearray,
|
|
217
|
+
callback: Callable[[MarketMessage], None],
|
|
218
|
+
) -> None:
|
|
180
219
|
"""
|
|
181
|
-
|
|
182
|
-
instance based on the message type indicator (first byte of the message).
|
|
183
|
-
|
|
184
|
-
All message type indicators are single ASCII characters.
|
|
220
|
+
Parses messages from data and invokes a callback for each message.
|
|
185
221
|
"""
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
)
|
|
222
|
+
parser_func = (
|
|
223
|
+
self.parse_stream
|
|
224
|
+
if isinstance(data, (bytes, bytearray))
|
|
225
|
+
else self.parse_file
|
|
226
|
+
)
|
|
227
|
+
for message in parser_func(data):
|
|
228
|
+
callback(message)
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: itchfeed
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.6
|
|
4
4
|
Summary: Simple parser for ITCH messages
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
License: The MIT License (MIT)
|
|
5
|
+
Author-email: Bertin Balouki SIMYELI <bertin@bbs-trading.com>
|
|
6
|
+
Maintainer-email: Bertin Balouki SIMYELI <bertin@bbs-trading.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/bbalouki/itch
|
|
9
|
+
Project-URL: Download, https://pypi.org/project/itchfeed/
|
|
11
10
|
Project-URL: Source Code, https://github.com/bbalouki/itch
|
|
12
11
|
Keywords: Finance,Financial,Quantitative,Equities,Totalview-ITCH,Totalview,Nasdaq-ITCH,Nasdaq,ITCH,Data,Feed,ETFs,Funds,Trading,Investing
|
|
13
12
|
Classifier: Development Status :: 5 - Production/Stable
|
|
@@ -19,20 +18,7 @@ Classifier: Operating System :: OS Independent
|
|
|
19
18
|
Description-Content-Type: text/markdown
|
|
20
19
|
License-File: LICENSE
|
|
21
20
|
Requires-Dist: pytest
|
|
22
|
-
Dynamic: author
|
|
23
|
-
Dynamic: author-email
|
|
24
|
-
Dynamic: classifier
|
|
25
|
-
Dynamic: description
|
|
26
|
-
Dynamic: description-content-type
|
|
27
|
-
Dynamic: download-url
|
|
28
|
-
Dynamic: home-page
|
|
29
|
-
Dynamic: keywords
|
|
30
|
-
Dynamic: license
|
|
31
21
|
Dynamic: license-file
|
|
32
|
-
Dynamic: maintainer
|
|
33
|
-
Dynamic: project-url
|
|
34
|
-
Dynamic: requires-dist
|
|
35
|
-
Dynamic: summary
|
|
36
22
|
|
|
37
23
|
# Nasdaq TotalView-ITCH 5.0 Parser
|
|
38
24
|
[](https://pypi.org/project/itchfeed/)
|
|
@@ -111,6 +97,8 @@ After installation (typically via pip), import the necessary modules directly in
|
|
|
111
97
|
|
|
112
98
|
## Usage
|
|
113
99
|
|
|
100
|
+
Download some sample data [here](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/)
|
|
101
|
+
|
|
114
102
|
### Parsing from a Binary File
|
|
115
103
|
|
|
116
104
|
This is useful for processing historical ITCH data stored in files. The `MessageParser` handles buffering efficiently.
|
|
@@ -131,9 +119,8 @@ parser = MessageParser() # Parses all messages by default
|
|
|
131
119
|
|
|
132
120
|
# Path to your ITCH 5.0 data file
|
|
133
121
|
itch_file_path = 'path/to/your/data'
|
|
134
|
-
# you can find sample data [here](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/)
|
|
135
122
|
|
|
136
|
-
# The `
|
|
123
|
+
# The `parse_file()` method reads the ITCH data in chunks.
|
|
137
124
|
# - `cachesize` (optional, default: 65536 bytes): This parameter determines the size of data chunks
|
|
138
125
|
# read from the file at a time. Adjusting this might impact performance for very large files
|
|
139
126
|
# or memory usage, but the default is generally suitable.
|
|
@@ -144,13 +131,9 @@ itch_file_path = 'path/to/your/data'
|
|
|
144
131
|
|
|
145
132
|
try:
|
|
146
133
|
with open(itch_file_path, 'rb') as itch_file:
|
|
147
|
-
#
|
|
148
|
-
parsed_messages = parser.read_message_from_file(itch_file) # You can also pass cachesize here, e.g., parser.read_message_from_file(itch_file, cachesize=131072)
|
|
149
|
-
|
|
150
|
-
print(f"Parsed {len(parsed_messages)} messages.")
|
|
151
|
-
|
|
134
|
+
# parse_file returns an Iterator of parsed message objects
|
|
152
135
|
# Process the messages
|
|
153
|
-
for message in
|
|
136
|
+
for message in parser.parse_file(itch_file):
|
|
154
137
|
# Access attributes directly
|
|
155
138
|
print(f"Type: {message.message_type.decode()}, Timestamp: {message.timestamp}")
|
|
156
139
|
|
|
@@ -192,14 +175,8 @@ parser = MessageParser()
|
|
|
192
175
|
# Example: \x00\x0bS...\x00\x25R...\x00\x27F...
|
|
193
176
|
raw_binary_data: bytes = b"..." # Your raw ITCH 5.0 data chunk
|
|
194
177
|
|
|
195
|
-
#
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
print(f"Parsed {message_queue.qsize()} messages from the byte chunk.")
|
|
199
|
-
|
|
200
|
-
# Process messages from the queue
|
|
201
|
-
while not message_queue.empty():
|
|
202
|
-
message = message_queue.get()
|
|
178
|
+
# parse_stream returns an Iterator of parsed message objects
|
|
179
|
+
for message in parser.parse_stream(raw_binary_data)
|
|
203
180
|
|
|
204
181
|
print(f"Type: {message.message_type.decode()}, Timestamp: {message.timestamp}")
|
|
205
182
|
|
|
@@ -462,11 +439,12 @@ Common scenarios that can lead to a `ValueError` include:
|
|
|
462
439
|
|
|
463
440
|
It's crucial to anticipate these errors in your application:
|
|
464
441
|
|
|
465
|
-
* **Use `try-except` Blocks:** Wrap your parsing calls (especially `
|
|
442
|
+
* **Use `try-except` Blocks:** Wrap your parsing calls (especially `parse_file` or `parse_stream`) in `try-except ValueError as e:` blocks.
|
|
466
443
|
```python
|
|
467
444
|
try:
|
|
468
445
|
# ... parsing operations ...
|
|
469
|
-
|
|
446
|
+
for message in parser.parse_file(itch_file):
|
|
447
|
+
...
|
|
470
448
|
except ValueError as e:
|
|
471
449
|
print(f"An error occurred during parsing: {e}")
|
|
472
450
|
# Log the error, problematic data chunk, or take other actions
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
LICENSE
|
|
2
2
|
MANIFEST.in
|
|
3
3
|
README.md
|
|
4
|
+
pyproject.toml
|
|
4
5
|
requirements.txt
|
|
5
6
|
setup.py
|
|
6
7
|
itch/__init__.py
|
|
@@ -13,4 +14,5 @@ itchfeed.egg-info/dependency_links.txt
|
|
|
13
14
|
itchfeed.egg-info/requires.txt
|
|
14
15
|
itchfeed.egg-info/top_level.txt
|
|
15
16
|
tests/test_create_message.py
|
|
16
|
-
tests/test_messages.py
|
|
17
|
+
tests/test_messages.py
|
|
18
|
+
tests/test_parser.py
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "itchfeed"
|
|
7
|
+
version = "1.0.6"
|
|
8
|
+
authors = [
|
|
9
|
+
{name = "Bertin Balouki SIMYELI", email = "bertin@bbs-trading.com"},
|
|
10
|
+
]
|
|
11
|
+
maintainers = [
|
|
12
|
+
{name = "Bertin Balouki SIMYELI", email = "bertin@bbs-trading.com"},
|
|
13
|
+
]
|
|
14
|
+
description = "Simple parser for ITCH messages"
|
|
15
|
+
readme = "README.md"
|
|
16
|
+
license = "MIT"
|
|
17
|
+
keywords = [
|
|
18
|
+
"Finance",
|
|
19
|
+
"Financial",
|
|
20
|
+
"Quantitative",
|
|
21
|
+
"Equities",
|
|
22
|
+
"Totalview-ITCH",
|
|
23
|
+
"Totalview",
|
|
24
|
+
"Nasdaq-ITCH",
|
|
25
|
+
"Nasdaq",
|
|
26
|
+
"ITCH",
|
|
27
|
+
"Data",
|
|
28
|
+
"Feed",
|
|
29
|
+
"ETFs",
|
|
30
|
+
"Funds",
|
|
31
|
+
"Trading",
|
|
32
|
+
"Investing",
|
|
33
|
+
]
|
|
34
|
+
classifiers = [
|
|
35
|
+
"Development Status :: 5 - Production/Stable",
|
|
36
|
+
"Intended Audience :: Developers",
|
|
37
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
38
|
+
"Topic :: Office/Business :: Financial :: Investment",
|
|
39
|
+
"Programming Language :: Python :: 3",
|
|
40
|
+
"Operating System :: OS Independent",
|
|
41
|
+
]
|
|
42
|
+
dynamic = ["dependencies"]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
"Homepage" = "https://github.com/bbalouki/itch"
|
|
46
|
+
"Download" = "https://pypi.org/project/itchfeed/"
|
|
47
|
+
"Source Code" = "https://github.com/bbalouki/itch"
|
|
48
|
+
|
|
49
|
+
[tool.setuptools]
|
|
50
|
+
packages = ["itch"]
|
|
51
|
+
|
|
52
|
+
[tool.setuptools.dynamic]
|
|
53
|
+
dependencies = { file = ["requirements.txt"] }
|
itchfeed-1.0.6/setup.py
ADDED
|
@@ -16,7 +16,7 @@ def test_create_message_and_pack(message_type, sample_data):
|
|
|
16
16
|
|
|
17
17
|
# Unpack the message using the original class constructor
|
|
18
18
|
message_class = messages[message_type]
|
|
19
|
-
unpacked_message = message_class(packed_message)
|
|
19
|
+
unpacked_message = message_class(packed_message) # type: ignore
|
|
20
20
|
|
|
21
21
|
# Verify that the attributes of the unpacked message match the original data
|
|
22
22
|
for key, expected_value in sample_data.items():
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from io import BytesIO
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from itch.messages import create_message
|
|
7
|
+
from itch.parser import MessageParser
|
|
8
|
+
from .data import TEST_DATA
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TestMessageParser(unittest.TestCase):
|
|
12
|
+
def setUp(self):
|
|
13
|
+
self.parser = MessageParser()
|
|
14
|
+
|
|
15
|
+
def test_parse_stream_single_message(self):
|
|
16
|
+
"""Test that the parser correctly parses a single message from a stream."""
|
|
17
|
+
raw_message = create_message(b"S", **TEST_DATA[b"S"]).to_bytes()
|
|
18
|
+
message_len = len(raw_message).to_bytes(1, "big")
|
|
19
|
+
data = b"\x00" + message_len + raw_message
|
|
20
|
+
messages = list(self.parser.parse_stream(data))
|
|
21
|
+
self.assertEqual(len(messages), 1)
|
|
22
|
+
self.assertIsInstance(
|
|
23
|
+
messages[0], type(create_message(b"S", **TEST_DATA[b"S"]))
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
def test_parse_stream_multiple_messages(self):
|
|
27
|
+
"""Test that the parser correctly parses multiple messages from a stream."""
|
|
28
|
+
raw_message_1 = create_message(b"S", **TEST_DATA[b"S"]).to_bytes()
|
|
29
|
+
raw_message_2 = create_message(b"A", **TEST_DATA[b"A"]).to_bytes()
|
|
30
|
+
message_len_1 = len(raw_message_1).to_bytes(1, "big")
|
|
31
|
+
message_len_2 = len(raw_message_2).to_bytes(1, "big")
|
|
32
|
+
data = (
|
|
33
|
+
b"\x00"
|
|
34
|
+
+ message_len_1
|
|
35
|
+
+ raw_message_1
|
|
36
|
+
+ b"\x00"
|
|
37
|
+
+ message_len_2
|
|
38
|
+
+ raw_message_2
|
|
39
|
+
)
|
|
40
|
+
messages = list(self.parser.parse_stream(data))
|
|
41
|
+
self.assertEqual(len(messages), 2)
|
|
42
|
+
self.assertIsInstance(
|
|
43
|
+
messages[0], type(create_message(b"S", **TEST_DATA[b"S"]))
|
|
44
|
+
)
|
|
45
|
+
self.assertIsInstance(
|
|
46
|
+
messages[1], type(create_message(b"A", **TEST_DATA[b"A"]))
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
def test_parse_file(self):
|
|
50
|
+
"""Test that the parser correctly parses messages from a file."""
|
|
51
|
+
raw_message = create_message(b"S", **TEST_DATA[b"S"]).to_bytes()
|
|
52
|
+
message_len = len(raw_message).to_bytes(1, "big")
|
|
53
|
+
data = b"\x00" + message_len + raw_message
|
|
54
|
+
file = BytesIO(data)
|
|
55
|
+
messages = list(self.parser.parse_file(file))
|
|
56
|
+
self.assertEqual(len(messages), 1)
|
|
57
|
+
self.assertIsInstance(
|
|
58
|
+
messages[0], type(create_message(b"S", **TEST_DATA[b"S"]))
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
def test_unknown_message_type(self):
|
|
62
|
+
"""Test that the parser raises a ValueError for an unknown message type."""
|
|
63
|
+
raw_message = b"\x00\x01\x00"
|
|
64
|
+
with self.assertRaises(ValueError):
|
|
65
|
+
list(self.parser.parse_stream(raw_message))
|
|
66
|
+
|
|
67
|
+
def test_incomplete_message(self):
|
|
68
|
+
"""Test that the parser handles incomplete messages."""
|
|
69
|
+
raw_message = create_message(b"S", **TEST_DATA[b"S"]).to_bytes()
|
|
70
|
+
message_len = len(raw_message).to_bytes(1, "big")
|
|
71
|
+
data = b"\x00" + message_len + raw_message[:-1]
|
|
72
|
+
messages = list(self.parser.parse_stream(data))
|
|
73
|
+
self.assertEqual(len(messages), 0)
|
|
74
|
+
|
|
75
|
+
def test_stop_on_system_event_c(self):
|
|
76
|
+
"""Test that the parser stops on a system event message with event code 'C'."""
|
|
77
|
+
kwargs = TEST_DATA[b"S"].copy()
|
|
78
|
+
kwargs["event_code"] = b"C"
|
|
79
|
+
raw_message_1 = create_message(b"S", **kwargs).to_bytes()
|
|
80
|
+
raw_message_2 = create_message(b"A", **TEST_DATA[b"A"]).to_bytes()
|
|
81
|
+
message_len_1 = len(raw_message_1).to_bytes(1, "big")
|
|
82
|
+
message_len_2 = len(raw_message_2).to_bytes(1, "big")
|
|
83
|
+
data = (
|
|
84
|
+
b"\x00"
|
|
85
|
+
+ message_len_1
|
|
86
|
+
+ raw_message_1
|
|
87
|
+
+ b"\x00"
|
|
88
|
+
+ message_len_2
|
|
89
|
+
+ raw_message_2
|
|
90
|
+
)
|
|
91
|
+
messages = list(self.parser.parse_stream(data))
|
|
92
|
+
self.assertEqual(len(messages), 1)
|
|
93
|
+
self.assertIsInstance(messages[0], type(create_message(b"S", **kwargs)))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@pytest.mark.parametrize("message_type", TEST_DATA.keys())
|
|
97
|
+
def test_parse_all_message_types(message_type):
|
|
98
|
+
"""Test that the parser correctly parses all message types."""
|
|
99
|
+
parser = MessageParser()
|
|
100
|
+
kwargs = TEST_DATA[message_type]
|
|
101
|
+
raw_message = create_message(message_type, **kwargs).to_bytes()
|
|
102
|
+
message_len = len(raw_message).to_bytes(1, "big")
|
|
103
|
+
data = b"\x00" + message_len + raw_message
|
|
104
|
+
messages = list(parser.parse_stream(data))
|
|
105
|
+
assert len(messages) == 1
|
|
106
|
+
assert isinstance(messages[0], type(create_message(message_type, **kwargs)))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
if __name__ == "__main__":
|
|
110
|
+
unittest.main()
|
itchfeed-1.0.5/itch/__init__.py
DELETED
itchfeed-1.0.5/setup.py
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
import io
|
|
2
|
-
from os import path
|
|
3
|
-
|
|
4
|
-
from setuptools import setup
|
|
5
|
-
|
|
6
|
-
here = path.abspath(path.dirname(__file__))
|
|
7
|
-
|
|
8
|
-
# Get the long description from the README file
|
|
9
|
-
with io.open(path.join(here, "README.md"), encoding="utf-8") as f:
|
|
10
|
-
long_description = f.read()
|
|
11
|
-
|
|
12
|
-
with io.open(path.join(here, "requirements.txt"), encoding="utf-8") as f:
|
|
13
|
-
REQUIREMENTS = [line.rstrip() for line in f]
|
|
14
|
-
|
|
15
|
-
VERSION = "1.0.5"
|
|
16
|
-
DESCRIPTION = "Simple parser for ITCH messages"
|
|
17
|
-
|
|
18
|
-
KEYWORDS = [
|
|
19
|
-
"Finance",
|
|
20
|
-
"Financial",
|
|
21
|
-
"Quantitative",
|
|
22
|
-
"Equities",
|
|
23
|
-
"Totalview-ITCH",
|
|
24
|
-
"Totalview",
|
|
25
|
-
"Nasdaq-ITCH",
|
|
26
|
-
"Nasdaq",
|
|
27
|
-
"ITCH",
|
|
28
|
-
"Data",
|
|
29
|
-
"Feed",
|
|
30
|
-
"ETFs",
|
|
31
|
-
"Funds",
|
|
32
|
-
"Trading",
|
|
33
|
-
"Investing",
|
|
34
|
-
]
|
|
35
|
-
|
|
36
|
-
CLASSIFIERS = [
|
|
37
|
-
"Development Status :: 5 - Production/Stable",
|
|
38
|
-
"Intended Audience :: Developers",
|
|
39
|
-
"Intended Audience :: Financial and Insurance Industry",
|
|
40
|
-
"Topic :: Office/Business :: Financial :: Investment",
|
|
41
|
-
"Programming Language :: Python :: 3",
|
|
42
|
-
"Operating System :: OS Independent",
|
|
43
|
-
]
|
|
44
|
-
|
|
45
|
-
INLCUDE = ["itch"]
|
|
46
|
-
EXCLUDE = ["tests"]
|
|
47
|
-
|
|
48
|
-
# Setting up
|
|
49
|
-
setup(
|
|
50
|
-
name="itchfeed",
|
|
51
|
-
version=VERSION,
|
|
52
|
-
author="Bertin Balouki SIMYELI",
|
|
53
|
-
url="https://github.com/bbalouki/itch",
|
|
54
|
-
download_url="https://pypi.org/project/itchfeed/",
|
|
55
|
-
project_urls={
|
|
56
|
-
"Source Code": "https://github.com/bbalouki/itch",
|
|
57
|
-
},
|
|
58
|
-
license="The MIT License (MIT)",
|
|
59
|
-
author_email="<bertin@bbstrader.com>",
|
|
60
|
-
maintainer="Bertin Balouki SIMYELI",
|
|
61
|
-
description=DESCRIPTION,
|
|
62
|
-
long_description=long_description,
|
|
63
|
-
long_description_content_type="text/markdown",
|
|
64
|
-
packages=INLCUDE,
|
|
65
|
-
install_requires=REQUIREMENTS,
|
|
66
|
-
keywords=KEYWORDS,
|
|
67
|
-
classifiers=CLASSIFIERS,
|
|
68
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|