itchfeed 1.0.3__tar.gz → 1.0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {itchfeed-1.0.3/itchfeed.egg-info → itchfeed-1.0.5}/PKG-INFO +85 -11
- {itchfeed-1.0.3 → itchfeed-1.0.5}/README.md +84 -10
- {itchfeed-1.0.3 → itchfeed-1.0.5}/itch/__init__.py +2 -1
- {itchfeed-1.0.3 → itchfeed-1.0.5}/itch/messages.py +103 -29
- itchfeed-1.0.5/itch/parser.py +192 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5/itchfeed.egg-info}/PKG-INFO +85 -11
- {itchfeed-1.0.3 → itchfeed-1.0.5}/itchfeed.egg-info/SOURCES.txt +1 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/setup.py +1 -1
- itchfeed-1.0.5/tests/test_create_message.py +28 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/tests/test_messages.py +5 -5
- itchfeed-1.0.3/itch/parser.py +0 -173
- {itchfeed-1.0.3 → itchfeed-1.0.5}/LICENSE +0 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/MANIFEST.in +0 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/itch/indicators.py +0 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/itchfeed.egg-info/dependency_links.txt +0 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/itchfeed.egg-info/requires.txt +0 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/itchfeed.egg-info/top_level.txt +0 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/requirements.txt +0 -0
- {itchfeed-1.0.3 → itchfeed-1.0.5}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: itchfeed
|
3
|
-
Version: 1.0.
|
3
|
+
Version: 1.0.5
|
4
4
|
Summary: Simple parser for ITCH messages
|
5
5
|
Home-page: https://github.com/bbalouki/itch
|
6
6
|
Download-URL: https://pypi.org/project/itchfeed/
|
@@ -52,12 +52,14 @@ Dynamic: summary
|
|
52
52
|
* [Usage](#usage)
|
53
53
|
* [Parsing from a Binary File](#parsing-from-a-binary-file)
|
54
54
|
* [Parsing from Raw Bytes](#parsing-from-raw-bytes)
|
55
|
+
* [Creating Messages Programmatically](#creating-messages-programmatically)
|
56
|
+
* [Example with Real-World Sample Data](#example-with-real-world-sample-data)
|
55
57
|
* [Interpreting Market Data (Conceptual Overview)](#interpreting-market-data-conceptual-overview)
|
56
58
|
* [Supported Message Types](#supported-message-types)
|
57
59
|
* [Data Representation](#data-representation)
|
58
60
|
* [Common Attributes of `MarketMessage`](#common-attributes-of-marketmessage)
|
59
61
|
* [Common Methods of `MarketMessage`](#common-methods-of-marketmessage)
|
60
|
-
* [Serializing Messages with `
|
62
|
+
* [Serializing Messages with `to_bytes()`](#serializing-messages-with-to_bytes)
|
61
63
|
* [Data Types in Parsed Messages](#data-types-in-parsed-messages)
|
62
64
|
* [Error Handling](#error-handling)
|
63
65
|
* [Handling Strategies](#handling-strategies)
|
@@ -168,8 +170,8 @@ try:
|
|
168
170
|
|
169
171
|
except FileNotFoundError:
|
170
172
|
print(f"Error: File not found at {itch_file_path}")
|
171
|
-
except
|
172
|
-
print(f"An error occurred: {e}")
|
173
|
+
except ValueError as e:
|
174
|
+
print(f"An error occurred during parsing: {e}")
|
173
175
|
|
174
176
|
```
|
175
177
|
|
@@ -211,6 +213,78 @@ while not message_queue.empty():
|
|
211
213
|
|
212
214
|
```
|
213
215
|
|
216
|
+
### Creating Messages Programmatically
|
217
|
+
|
218
|
+
In addition to parsing, `itch` provides a simple way to create ITCH message objects from scratch. This is particularly useful for:
|
219
|
+
- **Testing:** Generating specific message sequences to test your own downstream applications.
|
220
|
+
- **Simulation:** Building custom market simulators that produce ITCH-compliant data streams.
|
221
|
+
- **Data Generation:** Creating custom datasets for analysis or backtesting.
|
222
|
+
|
223
|
+
The `create_message` function is the primary tool for this purpose. It takes a `message_type` and keyword arguments corresponding to the desired message attributes.
|
224
|
+
|
225
|
+
Here's a basic example of how to create a `SystemEventMessage` to signal the "Start of Messages":
|
226
|
+
|
227
|
+
```python
|
228
|
+
from itch.messages import create_message, SystemEventMessage
|
229
|
+
|
230
|
+
# Define the attributes for the message
|
231
|
+
event_attributes = {
|
232
|
+
"stock_locate": 1,
|
233
|
+
"tracking_number": 2,
|
234
|
+
"timestamp": 1651500000 * 1_000_000_000,
|
235
|
+
"event_code": b"O"
|
236
|
+
}
|
237
|
+
|
238
|
+
# Create the message object
|
239
|
+
system_event_message = create_message(b"S", **event_attributes)
|
240
|
+
|
241
|
+
# You can now work with this object just like one from the parser
|
242
|
+
print(isinstance(system_event_message, SystemEventMessage))
|
243
|
+
# Expected output: True
|
244
|
+
|
245
|
+
print(system_event_message)
|
246
|
+
# Expected output: SystemEventMessage(description='System Event Message', event_code='O', message_format='!HHHIc', message_pack_format='!cHHHIc', message_size=12, message_type='S', price_precision=4, stock_locate=1, timestamp=86311638581248, tracking_number=2)
|
247
|
+
```
|
248
|
+
|
249
|
+
### Example with Real-World Sample Data
|
250
|
+
|
251
|
+
You can also use the sample data provided in `tests/data.py` to create messages, simulating a more realistic scenario.
|
252
|
+
|
253
|
+
```python
|
254
|
+
from itch.messages import create_message, AddOrderNoMPIAttributionMessage
|
255
|
+
from tests.data import TEST_DATA
|
256
|
+
|
257
|
+
# Get the sample data for an "Add Order" message (type 'A')
|
258
|
+
add_order_data = TEST_DATA[b"A"]
|
259
|
+
|
260
|
+
# Create the message
|
261
|
+
add_order_message = create_message(b"A", **add_order_data)
|
262
|
+
|
263
|
+
# Verify the type
|
264
|
+
print(isinstance(add_order_message, AddOrderNoMPIAttributionMessage))
|
265
|
+
# Expected output: True
|
266
|
+
|
267
|
+
# Access its attributes
|
268
|
+
print(f"Stock: {add_order_message.stock.decode().strip()}")
|
269
|
+
# Expected output: Stock: AAPL
|
270
|
+
|
271
|
+
print(f"Price: {add_order_message.decode_price('price')}")
|
272
|
+
# Expected output: Price: 150.1234
|
273
|
+
|
274
|
+
# Test all message types in the sample data
|
275
|
+
for message_type, sample_data in TEST_DATA.items():
|
276
|
+
print(f"Creating message of type {message_type}")
|
277
|
+
message = create_message(message_type, **sample_data)
|
278
|
+
print(f"Created message: {message}")
|
279
|
+
print(f"Packed message: {message.to_bytes()}")
|
280
|
+
print(f"Message size: {message.message_size}")
|
281
|
+
print(f"Message Attributes: {message.get_attributes()}")
|
282
|
+
assert len(message.to_bytes()) == message.message_size
|
283
|
+
print()
|
284
|
+
```
|
285
|
+
|
286
|
+
By leveraging `create_message`, you can build robust test suites for your trading algorithms, compliance checks, or data analysis pipelines without needing a live data feed.
|
287
|
+
|
214
288
|
## Interpreting Market Data (Conceptual Overview)
|
215
289
|
|
216
290
|
Parsing individual ITCH messages is the first step; understanding market dynamics often requires processing and correlating a sequence of these messages. This library provides the tools to decode messages, but interpreting their collective meaning requires building further logic.
|
@@ -314,14 +388,14 @@ The `MarketMessage` base class, and therefore all specific message classes, prov
|
|
314
388
|
* Returns a dictionary of all attributes (fields) of the message instance, along with their current values.
|
315
389
|
* This can be useful for generic inspection or logging of message contents without needing to know the specific type of the message beforehand.
|
316
390
|
|
317
|
-
### Serializing Messages with `
|
391
|
+
### Serializing Messages with `to_bytes()`
|
318
392
|
|
319
|
-
Each specific message class (e.g., `SystemEventMessage`, `AddOrderNoMPIAttributionMessage`) also provides a `
|
393
|
+
Each specific message class (e.g., `SystemEventMessage`, `AddOrderNoMPIAttributionMessage`) also provides a `to_bytes()` method. This method is the inverse of the parsing process.
|
320
394
|
|
321
395
|
* **Purpose:** It serializes the message object, with its current attribute values, back into its raw ITCH 5.0 binary format. The output is a `bytes` object representing the exact byte sequence that would appear in an ITCH data feed for that message.
|
322
396
|
* **Usefulness:**
|
323
397
|
* **Generating Test Data:** Create custom ITCH messages for testing your own ITCH processing applications.
|
324
|
-
* **Modifying Messages:** Parse an existing message, modify some of its attributes, and then `
|
398
|
+
* **Modifying Messages:** Parse an existing message, modify some of its attributes, and then `to_bytes()` it back into binary form.
|
325
399
|
* **Creating Custom ITCH Feeds:** While more involved, you could use this to construct sequences of ITCH messages for specialized scenarios.
|
326
400
|
|
327
401
|
**Example:**
|
@@ -341,21 +415,21 @@ import time
|
|
341
415
|
# for packing requires setting attributes manually if not using raw bytes for construction)
|
342
416
|
|
343
417
|
event_msg = SystemEventMessage.__new__(SystemEventMessage) # Create instance without calling __init__
|
344
|
-
event_msg.message_type = b'S' # Must be set for
|
418
|
+
event_msg.message_type = b'S' # Must be set for to_bytes() to know its type
|
345
419
|
event_msg.stock_locate = 0 # Placeholder or actual value
|
346
420
|
event_msg.tracking_number = 0 # Placeholder or actual value
|
347
421
|
event_msg.event_code = b'O' # Example: Start of Messages
|
348
422
|
|
349
423
|
# 2. Set the timestamp.
|
350
424
|
# The `timestamp` attribute (nanoseconds since midnight) must be set.
|
351
|
-
# The `
|
425
|
+
# The `to_bytes()` method will internally use `split_timestamp()` to get the parts.
|
352
426
|
current_nanoseconds = int(time.time() * 1e9) % (24 * 60 * 60 * int(1e9))
|
353
427
|
event_msg.timestamp = current_nanoseconds # Directly set the nanosecond timestamp
|
354
428
|
|
355
429
|
# 3. Pack the message into binary format.
|
356
|
-
# The
|
430
|
+
# The to_bytes() method prepends the message type and then packs stock_locate,
|
357
431
|
# tracking_number, the split timestamp, and then the message-specific fields.
|
358
|
-
packed_bytes = event_msg.
|
432
|
+
packed_bytes = event_msg.to_bytes()
|
359
433
|
|
360
434
|
# 4. The result is a bytes object
|
361
435
|
print(f"Packed {len(packed_bytes)} bytes: {packed_bytes.hex().upper()}")
|
@@ -16,12 +16,14 @@
|
|
16
16
|
* [Usage](#usage)
|
17
17
|
* [Parsing from a Binary File](#parsing-from-a-binary-file)
|
18
18
|
* [Parsing from Raw Bytes](#parsing-from-raw-bytes)
|
19
|
+
* [Creating Messages Programmatically](#creating-messages-programmatically)
|
20
|
+
* [Example with Real-World Sample Data](#example-with-real-world-sample-data)
|
19
21
|
* [Interpreting Market Data (Conceptual Overview)](#interpreting-market-data-conceptual-overview)
|
20
22
|
* [Supported Message Types](#supported-message-types)
|
21
23
|
* [Data Representation](#data-representation)
|
22
24
|
* [Common Attributes of `MarketMessage`](#common-attributes-of-marketmessage)
|
23
25
|
* [Common Methods of `MarketMessage`](#common-methods-of-marketmessage)
|
24
|
-
* [Serializing Messages with `
|
26
|
+
* [Serializing Messages with `to_bytes()`](#serializing-messages-with-to_bytes)
|
25
27
|
* [Data Types in Parsed Messages](#data-types-in-parsed-messages)
|
26
28
|
* [Error Handling](#error-handling)
|
27
29
|
* [Handling Strategies](#handling-strategies)
|
@@ -132,8 +134,8 @@ try:
|
|
132
134
|
|
133
135
|
except FileNotFoundError:
|
134
136
|
print(f"Error: File not found at {itch_file_path}")
|
135
|
-
except
|
136
|
-
print(f"An error occurred: {e}")
|
137
|
+
except ValueError as e:
|
138
|
+
print(f"An error occurred during parsing: {e}")
|
137
139
|
|
138
140
|
```
|
139
141
|
|
@@ -175,6 +177,78 @@ while not message_queue.empty():
|
|
175
177
|
|
176
178
|
```
|
177
179
|
|
180
|
+
### Creating Messages Programmatically
|
181
|
+
|
182
|
+
In addition to parsing, `itch` provides a simple way to create ITCH message objects from scratch. This is particularly useful for:
|
183
|
+
- **Testing:** Generating specific message sequences to test your own downstream applications.
|
184
|
+
- **Simulation:** Building custom market simulators that produce ITCH-compliant data streams.
|
185
|
+
- **Data Generation:** Creating custom datasets for analysis or backtesting.
|
186
|
+
|
187
|
+
The `create_message` function is the primary tool for this purpose. It takes a `message_type` and keyword arguments corresponding to the desired message attributes.
|
188
|
+
|
189
|
+
Here's a basic example of how to create a `SystemEventMessage` to signal the "Start of Messages":
|
190
|
+
|
191
|
+
```python
|
192
|
+
from itch.messages import create_message, SystemEventMessage
|
193
|
+
|
194
|
+
# Define the attributes for the message
|
195
|
+
event_attributes = {
|
196
|
+
"stock_locate": 1,
|
197
|
+
"tracking_number": 2,
|
198
|
+
"timestamp": 1651500000 * 1_000_000_000,
|
199
|
+
"event_code": b"O"
|
200
|
+
}
|
201
|
+
|
202
|
+
# Create the message object
|
203
|
+
system_event_message = create_message(b"S", **event_attributes)
|
204
|
+
|
205
|
+
# You can now work with this object just like one from the parser
|
206
|
+
print(isinstance(system_event_message, SystemEventMessage))
|
207
|
+
# Expected output: True
|
208
|
+
|
209
|
+
print(system_event_message)
|
210
|
+
# Expected output: SystemEventMessage(description='System Event Message', event_code='O', message_format='!HHHIc', message_pack_format='!cHHHIc', message_size=12, message_type='S', price_precision=4, stock_locate=1, timestamp=86311638581248, tracking_number=2)
|
211
|
+
```
|
212
|
+
|
213
|
+
### Example with Real-World Sample Data
|
214
|
+
|
215
|
+
You can also use the sample data provided in `tests/data.py` to create messages, simulating a more realistic scenario.
|
216
|
+
|
217
|
+
```python
|
218
|
+
from itch.messages import create_message, AddOrderNoMPIAttributionMessage
|
219
|
+
from tests.data import TEST_DATA
|
220
|
+
|
221
|
+
# Get the sample data for an "Add Order" message (type 'A')
|
222
|
+
add_order_data = TEST_DATA[b"A"]
|
223
|
+
|
224
|
+
# Create the message
|
225
|
+
add_order_message = create_message(b"A", **add_order_data)
|
226
|
+
|
227
|
+
# Verify the type
|
228
|
+
print(isinstance(add_order_message, AddOrderNoMPIAttributionMessage))
|
229
|
+
# Expected output: True
|
230
|
+
|
231
|
+
# Access its attributes
|
232
|
+
print(f"Stock: {add_order_message.stock.decode().strip()}")
|
233
|
+
# Expected output: Stock: AAPL
|
234
|
+
|
235
|
+
print(f"Price: {add_order_message.decode_price('price')}")
|
236
|
+
# Expected output: Price: 150.1234
|
237
|
+
|
238
|
+
# Test all message types in the sample data
|
239
|
+
for message_type, sample_data in TEST_DATA.items():
|
240
|
+
print(f"Creating message of type {message_type}")
|
241
|
+
message = create_message(message_type, **sample_data)
|
242
|
+
print(f"Created message: {message}")
|
243
|
+
print(f"Packed message: {message.to_bytes()}")
|
244
|
+
print(f"Message size: {message.message_size}")
|
245
|
+
print(f"Message Attributes: {message.get_attributes()}")
|
246
|
+
assert len(message.to_bytes()) == message.message_size
|
247
|
+
print()
|
248
|
+
```
|
249
|
+
|
250
|
+
By leveraging `create_message`, you can build robust test suites for your trading algorithms, compliance checks, or data analysis pipelines without needing a live data feed.
|
251
|
+
|
178
252
|
## Interpreting Market Data (Conceptual Overview)
|
179
253
|
|
180
254
|
Parsing individual ITCH messages is the first step; understanding market dynamics often requires processing and correlating a sequence of these messages. This library provides the tools to decode messages, but interpreting their collective meaning requires building further logic.
|
@@ -278,14 +352,14 @@ The `MarketMessage` base class, and therefore all specific message classes, prov
|
|
278
352
|
* Returns a dictionary of all attributes (fields) of the message instance, along with their current values.
|
279
353
|
* This can be useful for generic inspection or logging of message contents without needing to know the specific type of the message beforehand.
|
280
354
|
|
281
|
-
### Serializing Messages with `
|
355
|
+
### Serializing Messages with `to_bytes()`
|
282
356
|
|
283
|
-
Each specific message class (e.g., `SystemEventMessage`, `AddOrderNoMPIAttributionMessage`) also provides a `
|
357
|
+
Each specific message class (e.g., `SystemEventMessage`, `AddOrderNoMPIAttributionMessage`) also provides a `to_bytes()` method. This method is the inverse of the parsing process.
|
284
358
|
|
285
359
|
* **Purpose:** It serializes the message object, with its current attribute values, back into its raw ITCH 5.0 binary format. The output is a `bytes` object representing the exact byte sequence that would appear in an ITCH data feed for that message.
|
286
360
|
* **Usefulness:**
|
287
361
|
* **Generating Test Data:** Create custom ITCH messages for testing your own ITCH processing applications.
|
288
|
-
* **Modifying Messages:** Parse an existing message, modify some of its attributes, and then `
|
362
|
+
* **Modifying Messages:** Parse an existing message, modify some of its attributes, and then `to_bytes()` it back into binary form.
|
289
363
|
* **Creating Custom ITCH Feeds:** While more involved, you could use this to construct sequences of ITCH messages for specialized scenarios.
|
290
364
|
|
291
365
|
**Example:**
|
@@ -305,21 +379,21 @@ import time
|
|
305
379
|
# for packing requires setting attributes manually if not using raw bytes for construction)
|
306
380
|
|
307
381
|
event_msg = SystemEventMessage.__new__(SystemEventMessage) # Create instance without calling __init__
|
308
|
-
event_msg.message_type = b'S' # Must be set for
|
382
|
+
event_msg.message_type = b'S' # Must be set for to_bytes() to know its type
|
309
383
|
event_msg.stock_locate = 0 # Placeholder or actual value
|
310
384
|
event_msg.tracking_number = 0 # Placeholder or actual value
|
311
385
|
event_msg.event_code = b'O' # Example: Start of Messages
|
312
386
|
|
313
387
|
# 2. Set the timestamp.
|
314
388
|
# The `timestamp` attribute (nanoseconds since midnight) must be set.
|
315
|
-
# The `
|
389
|
+
# The `to_bytes()` method will internally use `split_timestamp()` to get the parts.
|
316
390
|
current_nanoseconds = int(time.time() * 1e9) % (24 * 60 * 60 * int(1e9))
|
317
391
|
event_msg.timestamp = current_nanoseconds # Directly set the nanosecond timestamp
|
318
392
|
|
319
393
|
# 3. Pack the message into binary format.
|
320
|
-
# The
|
394
|
+
# The to_bytes() method prepends the message type and then packs stock_locate,
|
321
395
|
# tracking_number, the split timestamp, and then the message-specific fields.
|
322
|
-
packed_bytes = event_msg.
|
396
|
+
packed_bytes = event_msg.to_bytes()
|
323
397
|
|
324
398
|
# 4. The result is a bytes object
|
325
399
|
print(f"Packed {len(packed_bytes)} bytes: {packed_bytes.hex().upper()}")
|
@@ -13,13 +13,21 @@ class MarketMessage(object):
|
|
13
13
|
|
14
14
|
All Message have the following attributes:
|
15
15
|
- message_type: A single letter that identify the message
|
16
|
+
- timestamp: Time at which the message was generated (Nanoseconds past midnight)
|
17
|
+
- stock_locate: Locate code identifying the security
|
18
|
+
- tracking_number: Nasdaq internal tracking number
|
19
|
+
|
20
|
+
The following attributes are not part of the message, but are used to describe the message:
|
16
21
|
- description: Describe the message
|
17
22
|
- message_format: string format using to unpack the message
|
18
23
|
- message_pack_format: string format using to pack the message
|
19
24
|
- message_size: The size in bytes of the message
|
20
|
-
|
21
|
-
|
22
|
-
|
25
|
+
|
26
|
+
# NOTE:
|
27
|
+
Prices are integers fields supplied with an associated precision. When converted to a decimal format, prices are in
|
28
|
+
fixed point format, where the precision defines the number of decimal places. For example, a field flagged as Price
|
29
|
+
(4) has an implied 4 decimal places. The maximum value of price (4) in TotalView ITCH is 200,000.0000 (decimal,
|
30
|
+
77359400 hex). ``price_precision`` is 4 for all messages except MWCBDeclineLeveMessage where ``price_precision`` is 8.
|
23
31
|
"""
|
24
32
|
|
25
33
|
message_type: bytes
|
@@ -32,9 +40,27 @@ class MarketMessage(object):
|
|
32
40
|
tracking_number: int
|
33
41
|
price_precision: int = 4
|
34
42
|
|
35
|
-
def __repr__(self):
|
43
|
+
def __repr__(self) -> str:
|
36
44
|
return repr(self.decode())
|
37
45
|
|
46
|
+
def __bytes__(self) -> bytes:
|
47
|
+
return self.to_bytes()
|
48
|
+
|
49
|
+
def to_bytes(self) -> bytes:
|
50
|
+
"""
|
51
|
+
Packs the message into bytes using the defined message_pack_format.
|
52
|
+
This method should be overridden by subclasses to include specific fields.
|
53
|
+
|
54
|
+
Note:
|
55
|
+
All packed messages do not include
|
56
|
+
- ``description``,
|
57
|
+
- ``message_format``,
|
58
|
+
- ``message_pack_format``,
|
59
|
+
- ``message_size``
|
60
|
+
- ``price_precision``
|
61
|
+
"""
|
62
|
+
pass
|
63
|
+
|
38
64
|
def set_timestamp(self, ts1: int, ts2: int):
|
39
65
|
"""
|
40
66
|
Reconstructs a 6-byte timestamp (48 bits) from two 32-bit unsigned integers.
|
@@ -159,7 +185,7 @@ class SystemEventMessage(MarketMessage):
|
|
159
185
|
) = struct.unpack(self.message_format, message[1:])
|
160
186
|
self.set_timestamp(timestamp1, timestamp2)
|
161
187
|
|
162
|
-
def
|
188
|
+
def to_bytes(self):
|
163
189
|
(timestamp1, timestamp2) = self.split_timestamp()
|
164
190
|
message = struct.pack(
|
165
191
|
self.message_pack_format,
|
@@ -281,7 +307,7 @@ class StockDirectoryMessage(MarketMessage):
|
|
281
307
|
) = struct.unpack(self.message_format, message[1:])
|
282
308
|
self.set_timestamp(timestamp1, timestamp2)
|
283
309
|
|
284
|
-
def
|
310
|
+
def to_bytes(self):
|
285
311
|
(timestamp1, timestamp2) = self.split_timestamp()
|
286
312
|
message = struct.pack(
|
287
313
|
self.message_pack_format,
|
@@ -361,7 +387,7 @@ class StockTradingActionMessage(MarketMessage):
|
|
361
387
|
) = struct.unpack(self.message_format, message[1:])
|
362
388
|
self.set_timestamp(timestamp1, timestamp2)
|
363
389
|
|
364
|
-
def
|
390
|
+
def to_bytes(self):
|
365
391
|
(timestamp1, timestamp2) = self.split_timestamp()
|
366
392
|
message = struct.pack(
|
367
393
|
self.message_pack_format,
|
@@ -422,7 +448,7 @@ class RegSHOMessage(MarketMessage):
|
|
422
448
|
) = struct.unpack(self.message_format, message[1:])
|
423
449
|
self.set_timestamp(timestamp1, timestamp2)
|
424
450
|
|
425
|
-
def
|
451
|
+
def to_bytes(self):
|
426
452
|
(timestamp1, timestamp2) = self.split_timestamp()
|
427
453
|
message = struct.pack(
|
428
454
|
self.message_pack_format,
|
@@ -484,7 +510,7 @@ class MarketParticipantPositionMessage(MarketMessage):
|
|
484
510
|
) = struct.unpack(self.message_format, message[1:])
|
485
511
|
self.set_timestamp(timestamp1, timestamp2)
|
486
512
|
|
487
|
-
def
|
513
|
+
def to_bytes(self):
|
488
514
|
(timestamp1, timestamp2) = self.split_timestamp()
|
489
515
|
message = struct.pack(
|
490
516
|
self.message_pack_format,
|
@@ -535,7 +561,7 @@ class MWCBDeclineLeveMessage(MarketMessage):
|
|
535
561
|
) = struct.unpack(self.message_format, message[1:])
|
536
562
|
self.set_timestamp(timestamp1, timestamp2)
|
537
563
|
|
538
|
-
def
|
564
|
+
def to_bytes(self):
|
539
565
|
(timestamp1, timestamp2) = self.split_timestamp()
|
540
566
|
message = struct.pack(
|
541
567
|
self.message_pack_format,
|
@@ -580,7 +606,7 @@ class MWCBStatusMessage(MarketMessage):
|
|
580
606
|
) = struct.unpack(self.message_format, message[1:])
|
581
607
|
self.set_timestamp(timestamp1, timestamp2)
|
582
608
|
|
583
|
-
def
|
609
|
+
def to_bytes(self):
|
584
610
|
(timestamp1, timestamp2) = self.split_timestamp()
|
585
611
|
message = struct.pack(
|
586
612
|
self.message_pack_format,
|
@@ -641,7 +667,7 @@ class IPOQuotingPeriodUpdateMessage(MarketMessage):
|
|
641
667
|
) = struct.unpack(self.message_format, message[1:])
|
642
668
|
self.set_timestamp(timestamp1, timestamp2)
|
643
669
|
|
644
|
-
def
|
670
|
+
def to_bytes(self):
|
645
671
|
(timestamp1, timestamp2) = self.split_timestamp()
|
646
672
|
message = struct.pack(
|
647
673
|
self.message_pack_format,
|
@@ -697,7 +723,7 @@ class LULDAuctionCollarMessage(MarketMessage):
|
|
697
723
|
) = struct.unpack(self.message_format, message[1:])
|
698
724
|
self.set_timestamp(timestamp1, timestamp2)
|
699
725
|
|
700
|
-
def
|
726
|
+
def to_bytes(self):
|
701
727
|
(timestamp1, timestamp2) = self.split_timestamp()
|
702
728
|
message = struct.pack(
|
703
729
|
self.message_pack_format,
|
@@ -762,7 +788,7 @@ class OperationalHaltMessage(MarketMessage):
|
|
762
788
|
) = struct.unpack(self.message_format, message[1:])
|
763
789
|
self.set_timestamp(timestamp1, timestamp2)
|
764
790
|
|
765
|
-
def
|
791
|
+
def to_bytes(self):
|
766
792
|
(timestamp1, timestamp2) = self.split_timestamp()
|
767
793
|
message = struct.pack(
|
768
794
|
self.message_pack_format,
|
@@ -825,7 +851,7 @@ class AddOrderNoMPIAttributionMessage(AddOrderMessage):
|
|
825
851
|
) = struct.unpack(self.message_format, message[1:])
|
826
852
|
self.set_timestamp(timestamp1, timestamp2)
|
827
853
|
|
828
|
-
def
|
854
|
+
def to_bytes(self):
|
829
855
|
(timestamp1, timestamp2) = self.split_timestamp()
|
830
856
|
message = struct.pack(
|
831
857
|
self.message_pack_format,
|
@@ -879,7 +905,7 @@ class AddOrderMPIDAttribution(AddOrderMessage):
|
|
879
905
|
) = struct.unpack(self.message_format, message[1:])
|
880
906
|
self.set_timestamp(timestamp1, timestamp2)
|
881
907
|
|
882
|
-
def
|
908
|
+
def to_bytes(self):
|
883
909
|
(timestamp1, timestamp2) = self.split_timestamp()
|
884
910
|
message = struct.pack(
|
885
911
|
self.message_pack_format,
|
@@ -950,7 +976,7 @@ class OrderExecutedMessage(ModifyOrderMessage):
|
|
950
976
|
) = struct.unpack(self.message_format, message[1:])
|
951
977
|
self.set_timestamp(timestamp1, timestamp2)
|
952
978
|
|
953
|
-
def
|
979
|
+
def to_bytes(self):
|
954
980
|
(timestamp1, timestamp2) = self.split_timestamp()
|
955
981
|
message = struct.pack(
|
956
982
|
self.message_pack_format,
|
@@ -1018,7 +1044,7 @@ class OrderExecutedWithPriceMessage(ModifyOrderMessage):
|
|
1018
1044
|
) = struct.unpack(self.message_format, message[1:])
|
1019
1045
|
self.set_timestamp(timestamp1, timestamp2)
|
1020
1046
|
|
1021
|
-
def
|
1047
|
+
def to_bytes(self):
|
1022
1048
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1023
1049
|
message = struct.pack(
|
1024
1050
|
self.message_pack_format,
|
@@ -1064,7 +1090,7 @@ class OrderCancelMessage(ModifyOrderMessage):
|
|
1064
1090
|
) = struct.unpack(self.message_format, message[1:])
|
1065
1091
|
self.set_timestamp(timestamp1, timestamp2)
|
1066
1092
|
|
1067
|
-
def
|
1093
|
+
def to_bytes(self):
|
1068
1094
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1069
1095
|
message = struct.pack(
|
1070
1096
|
self.message_pack_format,
|
@@ -1104,7 +1130,7 @@ class OrderDeleteMessage(ModifyOrderMessage):
|
|
1104
1130
|
) = struct.unpack(self.message_format, message[1:])
|
1105
1131
|
self.set_timestamp(timestamp1, timestamp2)
|
1106
1132
|
|
1107
|
-
def
|
1133
|
+
def to_bytes(self):
|
1108
1134
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1109
1135
|
message = struct.pack(
|
1110
1136
|
self.message_pack_format,
|
@@ -1157,7 +1183,7 @@ class OrderReplaceMessage(ModifyOrderMessage):
|
|
1157
1183
|
) = struct.unpack(self.message_format, message[1:])
|
1158
1184
|
self.set_timestamp(timestamp1, timestamp2)
|
1159
1185
|
|
1160
|
-
def
|
1186
|
+
def to_bytes(self):
|
1161
1187
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1162
1188
|
message = struct.pack(
|
1163
1189
|
self.message_pack_format,
|
@@ -1231,7 +1257,7 @@ class NonCrossTradeMessage(TradeMessage):
|
|
1231
1257
|
) = struct.unpack(self.message_format, message[1:])
|
1232
1258
|
self.set_timestamp(timestamp1, timestamp2)
|
1233
1259
|
|
1234
|
-
def
|
1260
|
+
def to_bytes(self):
|
1235
1261
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1236
1262
|
message = struct.pack(
|
1237
1263
|
self.message_pack_format,
|
@@ -1302,7 +1328,7 @@ class CrossTradeMessage(TradeMessage):
|
|
1302
1328
|
) = struct.unpack(self.message_format, message[1:])
|
1303
1329
|
self.set_timestamp(timestamp1, timestamp2)
|
1304
1330
|
|
1305
|
-
def
|
1331
|
+
def to_bytes(self):
|
1306
1332
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1307
1333
|
message = struct.pack(
|
1308
1334
|
self.message_pack_format,
|
@@ -1352,7 +1378,7 @@ class BrokenTradeMessage(TradeMessage):
|
|
1352
1378
|
) = struct.unpack(self.message_format, message[1:])
|
1353
1379
|
self.set_timestamp(timestamp1, timestamp2)
|
1354
1380
|
|
1355
|
-
def
|
1381
|
+
def to_bytes(self):
|
1356
1382
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1357
1383
|
message = struct.pack(
|
1358
1384
|
self.message_pack_format,
|
@@ -1434,7 +1460,7 @@ class NOIIMessage(MarketMessage):
|
|
1434
1460
|
) = struct.unpack(self.message_format, message[1:])
|
1435
1461
|
self.set_timestamp(timestamp1, timestamp2)
|
1436
1462
|
|
1437
|
-
def
|
1463
|
+
def to_bytes(self):
|
1438
1464
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1439
1465
|
message = struct.pack(
|
1440
1466
|
self.message_pack_format,
|
@@ -1489,7 +1515,7 @@ class RetailPriceImprovementIndicator(MarketMessage):
|
|
1489
1515
|
) = struct.unpack(self.message_format, message[1:])
|
1490
1516
|
self.set_timestamp(timestamp1, timestamp2)
|
1491
1517
|
|
1492
|
-
def
|
1518
|
+
def to_bytes(self):
|
1493
1519
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1494
1520
|
message = struct.pack(
|
1495
1521
|
self.message_pack_format,
|
@@ -1552,7 +1578,7 @@ class DLCRMessage(MarketMessage):
|
|
1552
1578
|
) = struct.unpack(self.message_format, message[1:])
|
1553
1579
|
self.set_timestamp(timestamp1, timestamp2)
|
1554
1580
|
|
1555
|
-
def
|
1581
|
+
def to_bytes(self):
|
1556
1582
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1557
1583
|
message = struct.pack(
|
1558
1584
|
self.message_pack_format,
|
@@ -1572,8 +1598,8 @@ class DLCRMessage(MarketMessage):
|
|
1572
1598
|
)
|
1573
1599
|
return message
|
1574
1600
|
|
1575
|
-
|
1576
|
-
messages
|
1601
|
+
messages: Dict[bytes, Type[MarketMessage]]
|
1602
|
+
messages = {
|
1577
1603
|
b"S": SystemEventMessage,
|
1578
1604
|
b"R": StockDirectoryMessage,
|
1579
1605
|
b"H": StockTradingActionMessage,
|
@@ -1598,3 +1624,51 @@ messages: Dict[bytes, Type[MarketMessage]] = {
|
|
1598
1624
|
b"N": RetailPriceImprovementIndicator,
|
1599
1625
|
b"O": DLCRMessage,
|
1600
1626
|
}
|
1627
|
+
|
1628
|
+
|
1629
|
+
def create_message(message_type: bytes, **kwargs) -> MarketMessage:
|
1630
|
+
"""
|
1631
|
+
Creates a new message of a given type with specified attributes.
|
1632
|
+
|
1633
|
+
This function simplifies the process of message creation by handling
|
1634
|
+
the instantiation and attribute setting for any valid message type.
|
1635
|
+
It's particularly useful for simulating trading environments or
|
1636
|
+
generating test data without manually packing and unpacking bytes.
|
1637
|
+
|
1638
|
+
Args:
|
1639
|
+
message_type (bytes):
|
1640
|
+
A single-byte identifier for the message type (e.g., b'A'
|
1641
|
+
for AddOrderNoMPIAttributionMessage).
|
1642
|
+
**kwargs:
|
1643
|
+
Keyword arguments representing the attributes of the message.
|
1644
|
+
These must match the attributes expected by the message class
|
1645
|
+
(e.g., `stock_locate`, `timestamp`, `price`).
|
1646
|
+
|
1647
|
+
Returns:
|
1648
|
+
MarketMessage:
|
1649
|
+
An instance of the corresponding message class, populated with
|
1650
|
+
the provided attributes.
|
1651
|
+
|
1652
|
+
Raises:
|
1653
|
+
ValueError:
|
1654
|
+
If the `message_type` is not found in the registered messages.
|
1655
|
+
"""
|
1656
|
+
message_class = messages.get(message_type)
|
1657
|
+
if not message_class:
|
1658
|
+
raise ValueError(f"Unknown message type: {message_type.decode()}")
|
1659
|
+
|
1660
|
+
# Create a new instance without calling __init__
|
1661
|
+
# __init__ is for unpacking, not creating
|
1662
|
+
instance = message_class.__new__(message_class)
|
1663
|
+
|
1664
|
+
# Set attributes from kwargs
|
1665
|
+
for key, value in kwargs.items():
|
1666
|
+
if key == 'timestamp':
|
1667
|
+
# Timestamps are 48-bit, so we need to mask the original value
|
1668
|
+
value &= ((1 << 48) - 1)
|
1669
|
+
setattr(instance, key, value)
|
1670
|
+
|
1671
|
+
# Set the message_type attribute on the instance, as it's used by pack()
|
1672
|
+
instance.message_type = message_type
|
1673
|
+
|
1674
|
+
return instance
|
@@ -0,0 +1,192 @@
|
|
1
|
+
from typing import IO, BinaryIO, Iterator
|
2
|
+
|
3
|
+
from itch.messages import MESSAGES, MarketMessage
|
4
|
+
from itch.messages import messages as msgs
|
5
|
+
|
6
|
+
|
7
|
+
class MessageParser(object):
|
8
|
+
"""
|
9
|
+
A market message parser for ITCH 5.0 data.
|
10
|
+
|
11
|
+
"""
|
12
|
+
|
13
|
+
def __init__(self, message_type: bytes = MESSAGES):
|
14
|
+
self.message_type = message_type
|
15
|
+
|
16
|
+
def read_message_from_file(
|
17
|
+
self, file: BinaryIO, cachesize: int = 65_536, save_file: IO = None
|
18
|
+
) -> Iterator[MarketMessage]:
|
19
|
+
"""
|
20
|
+
Reads and parses market messages from a binary file-like object.
|
21
|
+
|
22
|
+
This method processes binary data in chunks, extracts individual messages
|
23
|
+
according to a specific format, and returns a list of successfully decoded
|
24
|
+
MarketMessage objects. Parsing stops either when the end of the file is
|
25
|
+
reached or when a system message with an end-of-messages event code is encountered.
|
26
|
+
|
27
|
+
Args:
|
28
|
+
file (BinaryIO):
|
29
|
+
A binary file-like object (opened in binary mode) from which market messages are read.
|
30
|
+
cachesize (int, optional):
|
31
|
+
The size of each data chunk to read. Defaults to 65536 bytes (64KB).
|
32
|
+
save_file (IO, optional):
|
33
|
+
A binary file-like object (opened in binary write mode) where filtered messages are saved.
|
34
|
+
|
35
|
+
Yields:
|
36
|
+
MarketMessage:
|
37
|
+
The next parsed MarketMessage object from the file.
|
38
|
+
|
39
|
+
Raises:
|
40
|
+
ValueError:
|
41
|
+
If a message does not start with the expected 0x00 byte, indicating
|
42
|
+
an unexpected file format or possible corruption.
|
43
|
+
|
44
|
+
Notes:
|
45
|
+
- Each message starts with a 0x00 byte.
|
46
|
+
- The following byte specifies the message length.
|
47
|
+
- The complete message consists of the first 2 bytes and 'message length' bytes of body.
|
48
|
+
- If a system message (message_type == b'S') with event_code == b'C' is encountered,
|
49
|
+
parsing stops immediately.
|
50
|
+
|
51
|
+
Example:
|
52
|
+
>>> data_file = "01302020.NASDAQ_ITCH50.gz"
|
53
|
+
>>> message_type = b"AFE" # Example message type to filter
|
54
|
+
>>> parser = MessageParser(message_type=message_type)
|
55
|
+
>>> with gzip.open(data_file, "rb") as itch_file:
|
56
|
+
>>> message_count = 0
|
57
|
+
>>> start_time = time.time()
|
58
|
+
>>> for message in parser.read_message_from_file(itch_file):
|
59
|
+
>>> message_count += 1
|
60
|
+
>>> if message_count <= 5:
|
61
|
+
>>> print(message)
|
62
|
+
>>> end_time = time.time()
|
63
|
+
>>> print(f"Processed {message_count} messages in {end_time - start_time:.2f} seconds")
|
64
|
+
>>> print(f"Average time per message: {(end_time - start_time) / message_count:.6f} seconds")
|
65
|
+
"""
|
66
|
+
if not file.readable():
|
67
|
+
raise ValueError("file must be opened in binary read mode")
|
68
|
+
|
69
|
+
if save_file is not None:
|
70
|
+
if not save_file.writable():
|
71
|
+
raise ValueError("save_file must be opened in binary write mode")
|
72
|
+
|
73
|
+
data_buffer = b""
|
74
|
+
offset = 0
|
75
|
+
|
76
|
+
while True:
|
77
|
+
if len(data_buffer) - offset < 2:
|
78
|
+
data_buffer = data_buffer[offset:]
|
79
|
+
offset = 0
|
80
|
+
new_data = file.read(cachesize)
|
81
|
+
if not new_data:
|
82
|
+
break
|
83
|
+
data_buffer += new_data
|
84
|
+
|
85
|
+
if len(data_buffer) < 2:
|
86
|
+
break
|
87
|
+
|
88
|
+
if data_buffer[offset : offset + 1] != b"\x00":
|
89
|
+
raise ValueError(
|
90
|
+
"Unexpected byte: "
|
91
|
+
+ str(data_buffer[offset : offset + 1], encoding="ascii")
|
92
|
+
)
|
93
|
+
|
94
|
+
message_len = data_buffer[offset + 1]
|
95
|
+
total_len = 2 + message_len
|
96
|
+
|
97
|
+
if len(data_buffer) - offset < total_len:
|
98
|
+
data_buffer = data_buffer[offset:]
|
99
|
+
offset = 0
|
100
|
+
|
101
|
+
new_data = file.read(cachesize)
|
102
|
+
if not new_data:
|
103
|
+
break
|
104
|
+
data_buffer += new_data
|
105
|
+
continue
|
106
|
+
|
107
|
+
message_data = data_buffer[offset + 2 : offset + total_len]
|
108
|
+
message = self.get_message_type(message_data)
|
109
|
+
|
110
|
+
if message.message_type in self.message_type:
|
111
|
+
if save_file is not None:
|
112
|
+
msg_len_to_bytes = message.message_size.to_bytes()
|
113
|
+
save_file.write(b"\x00" + msg_len_to_bytes + message.to_bytes())
|
114
|
+
yield message
|
115
|
+
|
116
|
+
if message.message_type == b"S": # System message
|
117
|
+
if message.event_code == b"C": # End of messages
|
118
|
+
break
|
119
|
+
offset += total_len
|
120
|
+
|
121
|
+
def read_message_from_bytes(
|
122
|
+
self, data: bytes, save_file: IO = None
|
123
|
+
) -> Iterator[MarketMessage]:
|
124
|
+
"""
|
125
|
+
Process one or multiple ITCH binary messages from a raw bytes input.
|
126
|
+
|
127
|
+
Args:
|
128
|
+
data (bytes): Binary blob containing one or more ITCH messages.
|
129
|
+
|
130
|
+
save_file (IO, optional):
|
131
|
+
A binary file-like object (opened in binary write mode) where filtered messages are saved.
|
132
|
+
|
133
|
+
Yields:
|
134
|
+
MarketMessage:
|
135
|
+
The next parsed MarketMessage object from the bytes input.
|
136
|
+
|
137
|
+
Notes:
|
138
|
+
- Each message must be prefixed with a 0x00 header and a length byte.
|
139
|
+
- No buffering is done here — this is meant for real-time decoding.
|
140
|
+
"""
|
141
|
+
if not isinstance(data, (bytes, bytearray)):
|
142
|
+
raise TypeError("data must be bytes or bytearray not " + str(type(data)))
|
143
|
+
|
144
|
+
if save_file is not None:
|
145
|
+
if not save_file.writable():
|
146
|
+
raise ValueError("save_file must be opened in binary write mode")
|
147
|
+
|
148
|
+
offset = 0
|
149
|
+
data_view = memoryview(data)
|
150
|
+
data_len = len(data_view)
|
151
|
+
|
152
|
+
while offset + 2 <= data_len:
|
153
|
+
if data_view[offset : offset + 1] != b"\x00":
|
154
|
+
raise ValueError(
|
155
|
+
f"Unexpected start byte at offset {offset:offset+1}: "
|
156
|
+
f"{data_view[offset : offset + 1].tobytes()}"
|
157
|
+
)
|
158
|
+
msg_len = data_view[offset + 1]
|
159
|
+
total_len = 2 + msg_len
|
160
|
+
|
161
|
+
if offset + total_len > data_len:
|
162
|
+
break
|
163
|
+
|
164
|
+
raw_msg = data_view[offset + 2 : offset + total_len]
|
165
|
+
message = self.get_message_type(raw_msg.tobytes())
|
166
|
+
|
167
|
+
if message.message_type in self.message_type:
|
168
|
+
if save_file is not None:
|
169
|
+
msg_len_to_bytes = message.message_size.to_bytes()
|
170
|
+
save_file.write(b"\x00" + msg_len_to_bytes + message.to_bytes())
|
171
|
+
yield message
|
172
|
+
|
173
|
+
if message.message_type == b"S": # System message
|
174
|
+
if message.event_code == b"C": # End of messages
|
175
|
+
break
|
176
|
+
|
177
|
+
offset += total_len
|
178
|
+
|
179
|
+
def get_message_type(self, message: bytes) -> MarketMessage:
|
180
|
+
"""
|
181
|
+
Take an entire bytearray and return the appropriate ITCH message
|
182
|
+
instance based on the message type indicator (first byte of the message).
|
183
|
+
|
184
|
+
All message type indicators are single ASCII characters.
|
185
|
+
"""
|
186
|
+
message_type = message[0:1]
|
187
|
+
try:
|
188
|
+
return msgs[message_type](message)
|
189
|
+
except Exception:
|
190
|
+
raise ValueError(
|
191
|
+
f"Unknown message type: {message_type.decode(encoding='ascii')}"
|
192
|
+
)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: itchfeed
|
3
|
-
Version: 1.0.
|
3
|
+
Version: 1.0.5
|
4
4
|
Summary: Simple parser for ITCH messages
|
5
5
|
Home-page: https://github.com/bbalouki/itch
|
6
6
|
Download-URL: https://pypi.org/project/itchfeed/
|
@@ -52,12 +52,14 @@ Dynamic: summary
|
|
52
52
|
* [Usage](#usage)
|
53
53
|
* [Parsing from a Binary File](#parsing-from-a-binary-file)
|
54
54
|
* [Parsing from Raw Bytes](#parsing-from-raw-bytes)
|
55
|
+
* [Creating Messages Programmatically](#creating-messages-programmatically)
|
56
|
+
* [Example with Real-World Sample Data](#example-with-real-world-sample-data)
|
55
57
|
* [Interpreting Market Data (Conceptual Overview)](#interpreting-market-data-conceptual-overview)
|
56
58
|
* [Supported Message Types](#supported-message-types)
|
57
59
|
* [Data Representation](#data-representation)
|
58
60
|
* [Common Attributes of `MarketMessage`](#common-attributes-of-marketmessage)
|
59
61
|
* [Common Methods of `MarketMessage`](#common-methods-of-marketmessage)
|
60
|
-
* [Serializing Messages with `
|
62
|
+
* [Serializing Messages with `to_bytes()`](#serializing-messages-with-to_bytes)
|
61
63
|
* [Data Types in Parsed Messages](#data-types-in-parsed-messages)
|
62
64
|
* [Error Handling](#error-handling)
|
63
65
|
* [Handling Strategies](#handling-strategies)
|
@@ -168,8 +170,8 @@ try:
|
|
168
170
|
|
169
171
|
except FileNotFoundError:
|
170
172
|
print(f"Error: File not found at {itch_file_path}")
|
171
|
-
except
|
172
|
-
print(f"An error occurred: {e}")
|
173
|
+
except ValueError as e:
|
174
|
+
print(f"An error occurred during parsing: {e}")
|
173
175
|
|
174
176
|
```
|
175
177
|
|
@@ -211,6 +213,78 @@ while not message_queue.empty():
|
|
211
213
|
|
212
214
|
```
|
213
215
|
|
216
|
+
### Creating Messages Programmatically
|
217
|
+
|
218
|
+
In addition to parsing, `itch` provides a simple way to create ITCH message objects from scratch. This is particularly useful for:
|
219
|
+
- **Testing:** Generating specific message sequences to test your own downstream applications.
|
220
|
+
- **Simulation:** Building custom market simulators that produce ITCH-compliant data streams.
|
221
|
+
- **Data Generation:** Creating custom datasets for analysis or backtesting.
|
222
|
+
|
223
|
+
The `create_message` function is the primary tool for this purpose. It takes a `message_type` and keyword arguments corresponding to the desired message attributes.
|
224
|
+
|
225
|
+
Here's a basic example of how to create a `SystemEventMessage` to signal the "Start of Messages":
|
226
|
+
|
227
|
+
```python
|
228
|
+
from itch.messages import create_message, SystemEventMessage
|
229
|
+
|
230
|
+
# Define the attributes for the message
|
231
|
+
event_attributes = {
|
232
|
+
"stock_locate": 1,
|
233
|
+
"tracking_number": 2,
|
234
|
+
"timestamp": 1651500000 * 1_000_000_000,
|
235
|
+
"event_code": b"O"
|
236
|
+
}
|
237
|
+
|
238
|
+
# Create the message object
|
239
|
+
system_event_message = create_message(b"S", **event_attributes)
|
240
|
+
|
241
|
+
# You can now work with this object just like one from the parser
|
242
|
+
print(isinstance(system_event_message, SystemEventMessage))
|
243
|
+
# Expected output: True
|
244
|
+
|
245
|
+
print(system_event_message)
|
246
|
+
# Expected output: SystemEventMessage(description='System Event Message', event_code='O', message_format='!HHHIc', message_pack_format='!cHHHIc', message_size=12, message_type='S', price_precision=4, stock_locate=1, timestamp=86311638581248, tracking_number=2)
|
247
|
+
```
|
248
|
+
|
249
|
+
### Example with Real-World Sample Data
|
250
|
+
|
251
|
+
You can also use the sample data provided in `tests/data.py` to create messages, simulating a more realistic scenario.
|
252
|
+
|
253
|
+
```python
|
254
|
+
from itch.messages import create_message, AddOrderNoMPIAttributionMessage
|
255
|
+
from tests.data import TEST_DATA
|
256
|
+
|
257
|
+
# Get the sample data for an "Add Order" message (type 'A')
|
258
|
+
add_order_data = TEST_DATA[b"A"]
|
259
|
+
|
260
|
+
# Create the message
|
261
|
+
add_order_message = create_message(b"A", **add_order_data)
|
262
|
+
|
263
|
+
# Verify the type
|
264
|
+
print(isinstance(add_order_message, AddOrderNoMPIAttributionMessage))
|
265
|
+
# Expected output: True
|
266
|
+
|
267
|
+
# Access its attributes
|
268
|
+
print(f"Stock: {add_order_message.stock.decode().strip()}")
|
269
|
+
# Expected output: Stock: AAPL
|
270
|
+
|
271
|
+
print(f"Price: {add_order_message.decode_price('price')}")
|
272
|
+
# Expected output: Price: 150.1234
|
273
|
+
|
274
|
+
# Test all message types in the sample data
|
275
|
+
for message_type, sample_data in TEST_DATA.items():
|
276
|
+
print(f"Creating message of type {message_type}")
|
277
|
+
message = create_message(message_type, **sample_data)
|
278
|
+
print(f"Created message: {message}")
|
279
|
+
print(f"Packed message: {message.to_bytes()}")
|
280
|
+
print(f"Message size: {message.message_size}")
|
281
|
+
print(f"Message Attributes: {message.get_attributes()}")
|
282
|
+
assert len(message.to_bytes()) == message.message_size
|
283
|
+
print()
|
284
|
+
```
|
285
|
+
|
286
|
+
By leveraging `create_message`, you can build robust test suites for your trading algorithms, compliance checks, or data analysis pipelines without needing a live data feed.
|
287
|
+
|
214
288
|
## Interpreting Market Data (Conceptual Overview)
|
215
289
|
|
216
290
|
Parsing individual ITCH messages is the first step; understanding market dynamics often requires processing and correlating a sequence of these messages. This library provides the tools to decode messages, but interpreting their collective meaning requires building further logic.
|
@@ -314,14 +388,14 @@ The `MarketMessage` base class, and therefore all specific message classes, prov
|
|
314
388
|
* Returns a dictionary of all attributes (fields) of the message instance, along with their current values.
|
315
389
|
* This can be useful for generic inspection or logging of message contents without needing to know the specific type of the message beforehand.
|
316
390
|
|
317
|
-
### Serializing Messages with `
|
391
|
+
### Serializing Messages with `to_bytes()`
|
318
392
|
|
319
|
-
Each specific message class (e.g., `SystemEventMessage`, `AddOrderNoMPIAttributionMessage`) also provides a `
|
393
|
+
Each specific message class (e.g., `SystemEventMessage`, `AddOrderNoMPIAttributionMessage`) also provides a `to_bytes()` method. This method is the inverse of the parsing process.
|
320
394
|
|
321
395
|
* **Purpose:** It serializes the message object, with its current attribute values, back into its raw ITCH 5.0 binary format. The output is a `bytes` object representing the exact byte sequence that would appear in an ITCH data feed for that message.
|
322
396
|
* **Usefulness:**
|
323
397
|
* **Generating Test Data:** Create custom ITCH messages for testing your own ITCH processing applications.
|
324
|
-
* **Modifying Messages:** Parse an existing message, modify some of its attributes, and then `
|
398
|
+
* **Modifying Messages:** Parse an existing message, modify some of its attributes, and then `to_bytes()` it back into binary form.
|
325
399
|
* **Creating Custom ITCH Feeds:** While more involved, you could use this to construct sequences of ITCH messages for specialized scenarios.
|
326
400
|
|
327
401
|
**Example:**
|
@@ -341,21 +415,21 @@ import time
|
|
341
415
|
# for packing requires setting attributes manually if not using raw bytes for construction)
|
342
416
|
|
343
417
|
event_msg = SystemEventMessage.__new__(SystemEventMessage) # Create instance without calling __init__
|
344
|
-
event_msg.message_type = b'S' # Must be set for
|
418
|
+
event_msg.message_type = b'S' # Must be set for to_bytes() to know its type
|
345
419
|
event_msg.stock_locate = 0 # Placeholder or actual value
|
346
420
|
event_msg.tracking_number = 0 # Placeholder or actual value
|
347
421
|
event_msg.event_code = b'O' # Example: Start of Messages
|
348
422
|
|
349
423
|
# 2. Set the timestamp.
|
350
424
|
# The `timestamp` attribute (nanoseconds since midnight) must be set.
|
351
|
-
# The `
|
425
|
+
# The `to_bytes()` method will internally use `split_timestamp()` to get the parts.
|
352
426
|
current_nanoseconds = int(time.time() * 1e9) % (24 * 60 * 60 * int(1e9))
|
353
427
|
event_msg.timestamp = current_nanoseconds # Directly set the nanosecond timestamp
|
354
428
|
|
355
429
|
# 3. Pack the message into binary format.
|
356
|
-
# The
|
430
|
+
# The to_bytes() method prepends the message type and then packs stock_locate,
|
357
431
|
# tracking_number, the split timestamp, and then the message-specific fields.
|
358
|
-
packed_bytes = event_msg.
|
432
|
+
packed_bytes = event_msg.to_bytes()
|
359
433
|
|
360
434
|
# 4. The result is a bytes object
|
361
435
|
print(f"Packed {len(packed_bytes)} bytes: {packed_bytes.hex().upper()}")
|
@@ -12,7 +12,7 @@ with io.open(path.join(here, "README.md"), encoding="utf-8") as f:
|
|
12
12
|
with io.open(path.join(here, "requirements.txt"), encoding="utf-8") as f:
|
13
13
|
REQUIREMENTS = [line.rstrip() for line in f]
|
14
14
|
|
15
|
-
VERSION = "1.0.
|
15
|
+
VERSION = "1.0.5"
|
16
16
|
DESCRIPTION = "Simple parser for ITCH messages"
|
17
17
|
|
18
18
|
KEYWORDS = [
|
@@ -0,0 +1,28 @@
|
|
1
|
+
import pytest
|
2
|
+
from itch.messages import create_message, messages
|
3
|
+
from .data import TEST_DATA
|
4
|
+
|
5
|
+
@pytest.mark.parametrize("message_type, sample_data", TEST_DATA.items())
|
6
|
+
def test_create_message_and_pack(message_type, sample_data):
|
7
|
+
"""
|
8
|
+
Tests that create_message correctly creates a message that can be packed,
|
9
|
+
and that the packed message can be unpacked to match the original data.
|
10
|
+
"""
|
11
|
+
# Create a message using the new function
|
12
|
+
created_message = create_message(message_type, **sample_data)
|
13
|
+
|
14
|
+
# Pack the created message
|
15
|
+
packed_message = created_message.to_bytes()
|
16
|
+
|
17
|
+
# Unpack the message using the original class constructor
|
18
|
+
message_class = messages[message_type]
|
19
|
+
unpacked_message = message_class(packed_message)
|
20
|
+
|
21
|
+
# Verify that the attributes of the unpacked message match the original data
|
22
|
+
for key, expected_value in sample_data.items():
|
23
|
+
if key == 'timestamp':
|
24
|
+
# Timestamps are 48-bit, so we need to mask the original value
|
25
|
+
expected_value &= ((1 << 48) - 1)
|
26
|
+
assert getattr(unpacked_message, key) == expected_value, (
|
27
|
+
f"Attribute '{key}' mismatch for message type {message_type.decode()}"
|
28
|
+
)
|
@@ -76,7 +76,7 @@ def pack_message_data(
|
|
76
76
|
|
77
77
|
# Now call pack - instance should have all attributes defined in 'data'
|
78
78
|
try:
|
79
|
-
packed_message = instance.
|
79
|
+
packed_message = instance.to_bytes()
|
80
80
|
except struct.error as e:
|
81
81
|
print(f"Struct error during pack for {msg_class.__name__}: {e}")
|
82
82
|
raise
|
@@ -183,8 +183,8 @@ def test_market_message_decode_price():
|
|
183
183
|
msg.decode_price("stock")
|
184
184
|
|
185
185
|
# Test trying to decode a method
|
186
|
-
with pytest.raises(ValueError, match="Please check the price attribute for
|
187
|
-
msg.decode_price("
|
186
|
+
with pytest.raises(ValueError, match="Please check the price attribute for to_bytes"):
|
187
|
+
msg.decode_price("to_bytes")
|
188
188
|
|
189
189
|
|
190
190
|
# Helper Function for Verifying Unpacked Attributes
|
@@ -385,7 +385,7 @@ def test_pack_unpack_decode_consistency(message_params):
|
|
385
385
|
_verify_unpacked_attributes(instance, message_class, sample_data)
|
386
386
|
|
387
387
|
# Step 3: Pack the initialized object and compare
|
388
|
-
repacked_message = instance.
|
388
|
+
repacked_message = instance.to_bytes()
|
389
389
|
assert repacked_message == expected_packed_message, (
|
390
390
|
f"Repacked message does not match original for {message_class.__name__}"
|
391
391
|
)
|
@@ -427,7 +427,7 @@ def test_get_attributes(message_params):
|
|
427
427
|
assert "level1_price" in non_callable_attrs
|
428
428
|
|
429
429
|
# Check some expected callable attributes
|
430
|
-
assert "
|
430
|
+
assert "to_bytes" in callable_attrs
|
431
431
|
assert "decode" in callable_attrs
|
432
432
|
assert "set_timestamp" in callable_attrs
|
433
433
|
assert "split_timestamp" in callable_attrs
|
itchfeed-1.0.3/itch/parser.py
DELETED
@@ -1,173 +0,0 @@
|
|
1
|
-
from queue import Queue
|
2
|
-
from typing import BinaryIO, List, Type
|
3
|
-
|
4
|
-
from itch.messages import MESSAGES, MarketMessage
|
5
|
-
from itch.messages import messages as msgs
|
6
|
-
|
7
|
-
|
8
|
-
class MessageParser(object):
|
9
|
-
"""
|
10
|
-
A market message parser for ITCH 5.0 data.
|
11
|
-
|
12
|
-
"""
|
13
|
-
|
14
|
-
def __init__(self, message_type: bytes = MESSAGES):
|
15
|
-
self.message_type = message_type
|
16
|
-
|
17
|
-
def read_message_from_file(
|
18
|
-
self,
|
19
|
-
file: BinaryIO,
|
20
|
-
cachesize: int = 4096,
|
21
|
-
) -> List[MarketMessage]:
|
22
|
-
"""
|
23
|
-
Reads and parses market messages from a binary file-like object.
|
24
|
-
|
25
|
-
This method processes binary data in chunks, extracts individual messages
|
26
|
-
according to a specific format, and returns a list of successfully decoded
|
27
|
-
MarketMessage objects. Parsing stops either when the end of the file is
|
28
|
-
reached or when a system message with an end-of-messages event code is encountered.
|
29
|
-
|
30
|
-
Args:
|
31
|
-
file (BinaryIO):
|
32
|
-
A binary file-like object (opened in binary mode) from which market messages are read.
|
33
|
-
cachesize (int, optional):
|
34
|
-
The size (in bytes) of each data chunk read from the file. Defaults to 4096 bytes.
|
35
|
-
|
36
|
-
Returns:
|
37
|
-
List[MarketMessage]:
|
38
|
-
A list of parsed MarketMessage objects that match the allowed message types
|
39
|
-
defined in self.message_type.
|
40
|
-
|
41
|
-
Raises:
|
42
|
-
ValueError:
|
43
|
-
If a message does not start with the expected 0x00 byte, indicating
|
44
|
-
an unexpected file format or possible corruption.
|
45
|
-
|
46
|
-
Message Format:
|
47
|
-
- Each message starts with a 0x00 byte.
|
48
|
-
- The following byte specifies the message length.
|
49
|
-
- The complete message consists of the first 2 bytes and 'message length' bytes of body.
|
50
|
-
- If a system message (message_type == b'S') with event_code == b'C' is encountered,
|
51
|
-
parsing stops immediately.
|
52
|
-
|
53
|
-
Example:
|
54
|
-
>>> with open('market_data.bin', 'rb') as binary_file:
|
55
|
-
>>> messages = reader.read_message_from_file(binary_file, cachesize=4096)
|
56
|
-
>>> for message in messages:
|
57
|
-
>>> print(message)
|
58
|
-
"""
|
59
|
-
max_message_size = 52
|
60
|
-
file_end_reached = False
|
61
|
-
|
62
|
-
data_buffer = file.read(cachesize)
|
63
|
-
buffer_len = len(data_buffer)
|
64
|
-
messages: List[MarketMessage] = []
|
65
|
-
|
66
|
-
while not file_end_reached:
|
67
|
-
if buffer_len < 2:
|
68
|
-
new_data = file.read(cachesize)
|
69
|
-
if not new_data:
|
70
|
-
break
|
71
|
-
data_buffer += new_data
|
72
|
-
buffer_len = len(data_buffer)
|
73
|
-
continue
|
74
|
-
|
75
|
-
if data_buffer[0:1] != b"\x00":
|
76
|
-
raise ValueError(
|
77
|
-
"Unexpected byte: " + str(data_buffer[0:1], encoding="ascii")
|
78
|
-
)
|
79
|
-
|
80
|
-
message_len = data_buffer[1]
|
81
|
-
total_len = 2 + message_len
|
82
|
-
|
83
|
-
if buffer_len < total_len:
|
84
|
-
# Wait for more data if message is incomplete
|
85
|
-
new_data = file.read(cachesize)
|
86
|
-
if not new_data:
|
87
|
-
break
|
88
|
-
data_buffer += new_data
|
89
|
-
buffer_len = len(data_buffer)
|
90
|
-
continue
|
91
|
-
message_data = data_buffer[2:total_len]
|
92
|
-
message = self.get_message_type(message_data)
|
93
|
-
|
94
|
-
if message.message_type in self.message_type:
|
95
|
-
messages.append(message)
|
96
|
-
|
97
|
-
if message.message_type == b"S": # System message
|
98
|
-
if message.event_code == b"C": # End of messages
|
99
|
-
break
|
100
|
-
|
101
|
-
# Update buffer
|
102
|
-
data_buffer = data_buffer[total_len:]
|
103
|
-
buffer_len = len(data_buffer)
|
104
|
-
|
105
|
-
if buffer_len < max_message_size and not file_end_reached:
|
106
|
-
new_data = file.read(cachesize)
|
107
|
-
if not new_data:
|
108
|
-
file_end_reached = True
|
109
|
-
else:
|
110
|
-
data_buffer += new_data
|
111
|
-
buffer_len = len(data_buffer)
|
112
|
-
|
113
|
-
return messages
|
114
|
-
|
115
|
-
def read_message_from_bytes(self, data: bytes):
|
116
|
-
"""
|
117
|
-
Process one or multiple ITCH binary messages from a raw bytes input.
|
118
|
-
|
119
|
-
Args:
|
120
|
-
data (bytes): Binary blob containing one or more ITCH messages.
|
121
|
-
|
122
|
-
Returns:
|
123
|
-
Queue: A queue containing parsed ITCH message objects.
|
124
|
-
|
125
|
-
Notes:
|
126
|
-
- Each message must be prefixed with a 0x00 header and a length byte.
|
127
|
-
- No buffering is done here — this is meant for real-time decoding.
|
128
|
-
"""
|
129
|
-
|
130
|
-
offset = 0
|
131
|
-
messages = Queue()
|
132
|
-
while offset + 2 <= len(data):
|
133
|
-
# Each message starts with: 1-byte header (0x00) 1-byte length
|
134
|
-
if data[offset : offset + 1] != b"\x00":
|
135
|
-
raise ValueError(
|
136
|
-
f"Unexpected start byte at offset {offset}: "
|
137
|
-
f"{str(data[offset : offset + 1], encoding='ascii')}"
|
138
|
-
)
|
139
|
-
|
140
|
-
msg_len = data[offset + 1]
|
141
|
-
total_len = 2 + msg_len
|
142
|
-
|
143
|
-
if offset + total_len > len(data):
|
144
|
-
break
|
145
|
-
|
146
|
-
raw_msg = data[offset + 2 : offset + total_len]
|
147
|
-
message = self.get_message_type(raw_msg)
|
148
|
-
|
149
|
-
if message.message_type in self.message_type:
|
150
|
-
messages.put(message)
|
151
|
-
|
152
|
-
if message.message_type == b"S": # System message
|
153
|
-
if message.event_code == b"C": # End of messages
|
154
|
-
break
|
155
|
-
|
156
|
-
offset += total_len
|
157
|
-
|
158
|
-
return messages
|
159
|
-
|
160
|
-
def get_message_type(self, message: bytes) -> Type[MarketMessage]:
|
161
|
-
"""
|
162
|
-
Take an entire bytearray and return the appropriate ITCH message
|
163
|
-
instance based on the message type indicator (first byte of the message).
|
164
|
-
|
165
|
-
All message type indicators are single ASCII characters.
|
166
|
-
"""
|
167
|
-
message_type = message[0:1]
|
168
|
-
try:
|
169
|
-
return msgs[message_type](message)
|
170
|
-
except Exception:
|
171
|
-
raise ValueError(
|
172
|
-
f"Unknown message type: {message_type.decode(encoding='ascii')}"
|
173
|
-
)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|