itchfeed 1.0.3__py3-none-any.whl → 1.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- itch/__init__.py +2 -1
- itch/messages.py +103 -29
- itch/parser.py +92 -73
- {itchfeed-1.0.3.dist-info → itchfeed-1.0.5.dist-info}/METADATA +85 -11
- itchfeed-1.0.5.dist-info/RECORD +9 -0
- itchfeed-1.0.3.dist-info/RECORD +0 -9
- {itchfeed-1.0.3.dist-info → itchfeed-1.0.5.dist-info}/WHEEL +0 -0
- {itchfeed-1.0.3.dist-info → itchfeed-1.0.5.dist-info}/licenses/LICENSE +0 -0
- {itchfeed-1.0.3.dist-info → itchfeed-1.0.5.dist-info}/top_level.txt +0 -0
itch/__init__.py
CHANGED
itch/messages.py
CHANGED
@@ -13,13 +13,21 @@ class MarketMessage(object):
|
|
13
13
|
|
14
14
|
All Message have the following attributes:
|
15
15
|
- message_type: A single letter that identify the message
|
16
|
+
- timestamp: Time at which the message was generated (Nanoseconds past midnight)
|
17
|
+
- stock_locate: Locate code identifying the security
|
18
|
+
- tracking_number: Nasdaq internal tracking number
|
19
|
+
|
20
|
+
The following attributes are not part of the message, but are used to describe the message:
|
16
21
|
- description: Describe the message
|
17
22
|
- message_format: string format using to unpack the message
|
18
23
|
- message_pack_format: string format using to pack the message
|
19
24
|
- message_size: The size in bytes of the message
|
20
|
-
|
21
|
-
|
22
|
-
|
25
|
+
|
26
|
+
# NOTE:
|
27
|
+
Prices are integers fields supplied with an associated precision. When converted to a decimal format, prices are in
|
28
|
+
fixed point format, where the precision defines the number of decimal places. For example, a field flagged as Price
|
29
|
+
(4) has an implied 4 decimal places. The maximum value of price (4) in TotalView ITCH is 200,000.0000 (decimal,
|
30
|
+
77359400 hex). ``price_precision`` is 4 for all messages except MWCBDeclineLeveMessage where ``price_precision`` is 8.
|
23
31
|
"""
|
24
32
|
|
25
33
|
message_type: bytes
|
@@ -32,9 +40,27 @@ class MarketMessage(object):
|
|
32
40
|
tracking_number: int
|
33
41
|
price_precision: int = 4
|
34
42
|
|
35
|
-
def __repr__(self):
|
43
|
+
def __repr__(self) -> str:
|
36
44
|
return repr(self.decode())
|
37
45
|
|
46
|
+
def __bytes__(self) -> bytes:
|
47
|
+
return self.to_bytes()
|
48
|
+
|
49
|
+
def to_bytes(self) -> bytes:
|
50
|
+
"""
|
51
|
+
Packs the message into bytes using the defined message_pack_format.
|
52
|
+
This method should be overridden by subclasses to include specific fields.
|
53
|
+
|
54
|
+
Note:
|
55
|
+
All packed messages do not include
|
56
|
+
- ``description``,
|
57
|
+
- ``message_format``,
|
58
|
+
- ``message_pack_format``,
|
59
|
+
- ``message_size``
|
60
|
+
- ``price_precision``
|
61
|
+
"""
|
62
|
+
pass
|
63
|
+
|
38
64
|
def set_timestamp(self, ts1: int, ts2: int):
|
39
65
|
"""
|
40
66
|
Reconstructs a 6-byte timestamp (48 bits) from two 32-bit unsigned integers.
|
@@ -159,7 +185,7 @@ class SystemEventMessage(MarketMessage):
|
|
159
185
|
) = struct.unpack(self.message_format, message[1:])
|
160
186
|
self.set_timestamp(timestamp1, timestamp2)
|
161
187
|
|
162
|
-
def
|
188
|
+
def to_bytes(self):
|
163
189
|
(timestamp1, timestamp2) = self.split_timestamp()
|
164
190
|
message = struct.pack(
|
165
191
|
self.message_pack_format,
|
@@ -281,7 +307,7 @@ class StockDirectoryMessage(MarketMessage):
|
|
281
307
|
) = struct.unpack(self.message_format, message[1:])
|
282
308
|
self.set_timestamp(timestamp1, timestamp2)
|
283
309
|
|
284
|
-
def
|
310
|
+
def to_bytes(self):
|
285
311
|
(timestamp1, timestamp2) = self.split_timestamp()
|
286
312
|
message = struct.pack(
|
287
313
|
self.message_pack_format,
|
@@ -361,7 +387,7 @@ class StockTradingActionMessage(MarketMessage):
|
|
361
387
|
) = struct.unpack(self.message_format, message[1:])
|
362
388
|
self.set_timestamp(timestamp1, timestamp2)
|
363
389
|
|
364
|
-
def
|
390
|
+
def to_bytes(self):
|
365
391
|
(timestamp1, timestamp2) = self.split_timestamp()
|
366
392
|
message = struct.pack(
|
367
393
|
self.message_pack_format,
|
@@ -422,7 +448,7 @@ class RegSHOMessage(MarketMessage):
|
|
422
448
|
) = struct.unpack(self.message_format, message[1:])
|
423
449
|
self.set_timestamp(timestamp1, timestamp2)
|
424
450
|
|
425
|
-
def
|
451
|
+
def to_bytes(self):
|
426
452
|
(timestamp1, timestamp2) = self.split_timestamp()
|
427
453
|
message = struct.pack(
|
428
454
|
self.message_pack_format,
|
@@ -484,7 +510,7 @@ class MarketParticipantPositionMessage(MarketMessage):
|
|
484
510
|
) = struct.unpack(self.message_format, message[1:])
|
485
511
|
self.set_timestamp(timestamp1, timestamp2)
|
486
512
|
|
487
|
-
def
|
513
|
+
def to_bytes(self):
|
488
514
|
(timestamp1, timestamp2) = self.split_timestamp()
|
489
515
|
message = struct.pack(
|
490
516
|
self.message_pack_format,
|
@@ -535,7 +561,7 @@ class MWCBDeclineLeveMessage(MarketMessage):
|
|
535
561
|
) = struct.unpack(self.message_format, message[1:])
|
536
562
|
self.set_timestamp(timestamp1, timestamp2)
|
537
563
|
|
538
|
-
def
|
564
|
+
def to_bytes(self):
|
539
565
|
(timestamp1, timestamp2) = self.split_timestamp()
|
540
566
|
message = struct.pack(
|
541
567
|
self.message_pack_format,
|
@@ -580,7 +606,7 @@ class MWCBStatusMessage(MarketMessage):
|
|
580
606
|
) = struct.unpack(self.message_format, message[1:])
|
581
607
|
self.set_timestamp(timestamp1, timestamp2)
|
582
608
|
|
583
|
-
def
|
609
|
+
def to_bytes(self):
|
584
610
|
(timestamp1, timestamp2) = self.split_timestamp()
|
585
611
|
message = struct.pack(
|
586
612
|
self.message_pack_format,
|
@@ -641,7 +667,7 @@ class IPOQuotingPeriodUpdateMessage(MarketMessage):
|
|
641
667
|
) = struct.unpack(self.message_format, message[1:])
|
642
668
|
self.set_timestamp(timestamp1, timestamp2)
|
643
669
|
|
644
|
-
def
|
670
|
+
def to_bytes(self):
|
645
671
|
(timestamp1, timestamp2) = self.split_timestamp()
|
646
672
|
message = struct.pack(
|
647
673
|
self.message_pack_format,
|
@@ -697,7 +723,7 @@ class LULDAuctionCollarMessage(MarketMessage):
|
|
697
723
|
) = struct.unpack(self.message_format, message[1:])
|
698
724
|
self.set_timestamp(timestamp1, timestamp2)
|
699
725
|
|
700
|
-
def
|
726
|
+
def to_bytes(self):
|
701
727
|
(timestamp1, timestamp2) = self.split_timestamp()
|
702
728
|
message = struct.pack(
|
703
729
|
self.message_pack_format,
|
@@ -762,7 +788,7 @@ class OperationalHaltMessage(MarketMessage):
|
|
762
788
|
) = struct.unpack(self.message_format, message[1:])
|
763
789
|
self.set_timestamp(timestamp1, timestamp2)
|
764
790
|
|
765
|
-
def
|
791
|
+
def to_bytes(self):
|
766
792
|
(timestamp1, timestamp2) = self.split_timestamp()
|
767
793
|
message = struct.pack(
|
768
794
|
self.message_pack_format,
|
@@ -825,7 +851,7 @@ class AddOrderNoMPIAttributionMessage(AddOrderMessage):
|
|
825
851
|
) = struct.unpack(self.message_format, message[1:])
|
826
852
|
self.set_timestamp(timestamp1, timestamp2)
|
827
853
|
|
828
|
-
def
|
854
|
+
def to_bytes(self):
|
829
855
|
(timestamp1, timestamp2) = self.split_timestamp()
|
830
856
|
message = struct.pack(
|
831
857
|
self.message_pack_format,
|
@@ -879,7 +905,7 @@ class AddOrderMPIDAttribution(AddOrderMessage):
|
|
879
905
|
) = struct.unpack(self.message_format, message[1:])
|
880
906
|
self.set_timestamp(timestamp1, timestamp2)
|
881
907
|
|
882
|
-
def
|
908
|
+
def to_bytes(self):
|
883
909
|
(timestamp1, timestamp2) = self.split_timestamp()
|
884
910
|
message = struct.pack(
|
885
911
|
self.message_pack_format,
|
@@ -950,7 +976,7 @@ class OrderExecutedMessage(ModifyOrderMessage):
|
|
950
976
|
) = struct.unpack(self.message_format, message[1:])
|
951
977
|
self.set_timestamp(timestamp1, timestamp2)
|
952
978
|
|
953
|
-
def
|
979
|
+
def to_bytes(self):
|
954
980
|
(timestamp1, timestamp2) = self.split_timestamp()
|
955
981
|
message = struct.pack(
|
956
982
|
self.message_pack_format,
|
@@ -1018,7 +1044,7 @@ class OrderExecutedWithPriceMessage(ModifyOrderMessage):
|
|
1018
1044
|
) = struct.unpack(self.message_format, message[1:])
|
1019
1045
|
self.set_timestamp(timestamp1, timestamp2)
|
1020
1046
|
|
1021
|
-
def
|
1047
|
+
def to_bytes(self):
|
1022
1048
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1023
1049
|
message = struct.pack(
|
1024
1050
|
self.message_pack_format,
|
@@ -1064,7 +1090,7 @@ class OrderCancelMessage(ModifyOrderMessage):
|
|
1064
1090
|
) = struct.unpack(self.message_format, message[1:])
|
1065
1091
|
self.set_timestamp(timestamp1, timestamp2)
|
1066
1092
|
|
1067
|
-
def
|
1093
|
+
def to_bytes(self):
|
1068
1094
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1069
1095
|
message = struct.pack(
|
1070
1096
|
self.message_pack_format,
|
@@ -1104,7 +1130,7 @@ class OrderDeleteMessage(ModifyOrderMessage):
|
|
1104
1130
|
) = struct.unpack(self.message_format, message[1:])
|
1105
1131
|
self.set_timestamp(timestamp1, timestamp2)
|
1106
1132
|
|
1107
|
-
def
|
1133
|
+
def to_bytes(self):
|
1108
1134
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1109
1135
|
message = struct.pack(
|
1110
1136
|
self.message_pack_format,
|
@@ -1157,7 +1183,7 @@ class OrderReplaceMessage(ModifyOrderMessage):
|
|
1157
1183
|
) = struct.unpack(self.message_format, message[1:])
|
1158
1184
|
self.set_timestamp(timestamp1, timestamp2)
|
1159
1185
|
|
1160
|
-
def
|
1186
|
+
def to_bytes(self):
|
1161
1187
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1162
1188
|
message = struct.pack(
|
1163
1189
|
self.message_pack_format,
|
@@ -1231,7 +1257,7 @@ class NonCrossTradeMessage(TradeMessage):
|
|
1231
1257
|
) = struct.unpack(self.message_format, message[1:])
|
1232
1258
|
self.set_timestamp(timestamp1, timestamp2)
|
1233
1259
|
|
1234
|
-
def
|
1260
|
+
def to_bytes(self):
|
1235
1261
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1236
1262
|
message = struct.pack(
|
1237
1263
|
self.message_pack_format,
|
@@ -1302,7 +1328,7 @@ class CrossTradeMessage(TradeMessage):
|
|
1302
1328
|
) = struct.unpack(self.message_format, message[1:])
|
1303
1329
|
self.set_timestamp(timestamp1, timestamp2)
|
1304
1330
|
|
1305
|
-
def
|
1331
|
+
def to_bytes(self):
|
1306
1332
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1307
1333
|
message = struct.pack(
|
1308
1334
|
self.message_pack_format,
|
@@ -1352,7 +1378,7 @@ class BrokenTradeMessage(TradeMessage):
|
|
1352
1378
|
) = struct.unpack(self.message_format, message[1:])
|
1353
1379
|
self.set_timestamp(timestamp1, timestamp2)
|
1354
1380
|
|
1355
|
-
def
|
1381
|
+
def to_bytes(self):
|
1356
1382
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1357
1383
|
message = struct.pack(
|
1358
1384
|
self.message_pack_format,
|
@@ -1434,7 +1460,7 @@ class NOIIMessage(MarketMessage):
|
|
1434
1460
|
) = struct.unpack(self.message_format, message[1:])
|
1435
1461
|
self.set_timestamp(timestamp1, timestamp2)
|
1436
1462
|
|
1437
|
-
def
|
1463
|
+
def to_bytes(self):
|
1438
1464
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1439
1465
|
message = struct.pack(
|
1440
1466
|
self.message_pack_format,
|
@@ -1489,7 +1515,7 @@ class RetailPriceImprovementIndicator(MarketMessage):
|
|
1489
1515
|
) = struct.unpack(self.message_format, message[1:])
|
1490
1516
|
self.set_timestamp(timestamp1, timestamp2)
|
1491
1517
|
|
1492
|
-
def
|
1518
|
+
def to_bytes(self):
|
1493
1519
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1494
1520
|
message = struct.pack(
|
1495
1521
|
self.message_pack_format,
|
@@ -1552,7 +1578,7 @@ class DLCRMessage(MarketMessage):
|
|
1552
1578
|
) = struct.unpack(self.message_format, message[1:])
|
1553
1579
|
self.set_timestamp(timestamp1, timestamp2)
|
1554
1580
|
|
1555
|
-
def
|
1581
|
+
def to_bytes(self):
|
1556
1582
|
(timestamp1, timestamp2) = self.split_timestamp()
|
1557
1583
|
message = struct.pack(
|
1558
1584
|
self.message_pack_format,
|
@@ -1572,8 +1598,8 @@ class DLCRMessage(MarketMessage):
|
|
1572
1598
|
)
|
1573
1599
|
return message
|
1574
1600
|
|
1575
|
-
|
1576
|
-
messages
|
1601
|
+
messages: Dict[bytes, Type[MarketMessage]]
|
1602
|
+
messages = {
|
1577
1603
|
b"S": SystemEventMessage,
|
1578
1604
|
b"R": StockDirectoryMessage,
|
1579
1605
|
b"H": StockTradingActionMessage,
|
@@ -1598,3 +1624,51 @@ messages: Dict[bytes, Type[MarketMessage]] = {
|
|
1598
1624
|
b"N": RetailPriceImprovementIndicator,
|
1599
1625
|
b"O": DLCRMessage,
|
1600
1626
|
}
|
1627
|
+
|
1628
|
+
|
1629
|
+
def create_message(message_type: bytes, **kwargs) -> MarketMessage:
|
1630
|
+
"""
|
1631
|
+
Creates a new message of a given type with specified attributes.
|
1632
|
+
|
1633
|
+
This function simplifies the process of message creation by handling
|
1634
|
+
the instantiation and attribute setting for any valid message type.
|
1635
|
+
It's particularly useful for simulating trading environments or
|
1636
|
+
generating test data without manually packing and unpacking bytes.
|
1637
|
+
|
1638
|
+
Args:
|
1639
|
+
message_type (bytes):
|
1640
|
+
A single-byte identifier for the message type (e.g., b'A'
|
1641
|
+
for AddOrderNoMPIAttributionMessage).
|
1642
|
+
**kwargs:
|
1643
|
+
Keyword arguments representing the attributes of the message.
|
1644
|
+
These must match the attributes expected by the message class
|
1645
|
+
(e.g., `stock_locate`, `timestamp`, `price`).
|
1646
|
+
|
1647
|
+
Returns:
|
1648
|
+
MarketMessage:
|
1649
|
+
An instance of the corresponding message class, populated with
|
1650
|
+
the provided attributes.
|
1651
|
+
|
1652
|
+
Raises:
|
1653
|
+
ValueError:
|
1654
|
+
If the `message_type` is not found in the registered messages.
|
1655
|
+
"""
|
1656
|
+
message_class = messages.get(message_type)
|
1657
|
+
if not message_class:
|
1658
|
+
raise ValueError(f"Unknown message type: {message_type.decode()}")
|
1659
|
+
|
1660
|
+
# Create a new instance without calling __init__
|
1661
|
+
# __init__ is for unpacking, not creating
|
1662
|
+
instance = message_class.__new__(message_class)
|
1663
|
+
|
1664
|
+
# Set attributes from kwargs
|
1665
|
+
for key, value in kwargs.items():
|
1666
|
+
if key == 'timestamp':
|
1667
|
+
# Timestamps are 48-bit, so we need to mask the original value
|
1668
|
+
value &= ((1 << 48) - 1)
|
1669
|
+
setattr(instance, key, value)
|
1670
|
+
|
1671
|
+
# Set the message_type attribute on the instance, as it's used by pack()
|
1672
|
+
instance.message_type = message_type
|
1673
|
+
|
1674
|
+
return instance
|
itch/parser.py
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
from
|
2
|
-
from typing import BinaryIO, List, Type
|
1
|
+
from typing import IO, BinaryIO, Iterator
|
3
2
|
|
4
3
|
from itch.messages import MESSAGES, MarketMessage
|
5
4
|
from itch.messages import messages as msgs
|
@@ -15,139 +14,161 @@ class MessageParser(object):
|
|
15
14
|
self.message_type = message_type
|
16
15
|
|
17
16
|
def read_message_from_file(
|
18
|
-
self,
|
19
|
-
|
20
|
-
cachesize: int = 4096,
|
21
|
-
) -> List[MarketMessage]:
|
17
|
+
self, file: BinaryIO, cachesize: int = 65_536, save_file: IO = None
|
18
|
+
) -> Iterator[MarketMessage]:
|
22
19
|
"""
|
23
20
|
Reads and parses market messages from a binary file-like object.
|
24
21
|
|
25
22
|
This method processes binary data in chunks, extracts individual messages
|
26
|
-
according to a specific format, and returns a list of successfully decoded
|
27
|
-
MarketMessage objects. Parsing stops either when the end of the file is
|
23
|
+
according to a specific format, and returns a list of successfully decoded
|
24
|
+
MarketMessage objects. Parsing stops either when the end of the file is
|
28
25
|
reached or when a system message with an end-of-messages event code is encountered.
|
29
26
|
|
30
27
|
Args:
|
31
|
-
file (BinaryIO):
|
28
|
+
file (BinaryIO):
|
32
29
|
A binary file-like object (opened in binary mode) from which market messages are read.
|
33
|
-
cachesize (int, optional):
|
34
|
-
The size
|
30
|
+
cachesize (int, optional):
|
31
|
+
The size of each data chunk to read. Defaults to 65536 bytes (64KB).
|
32
|
+
save_file (IO, optional):
|
33
|
+
A binary file-like object (opened in binary write mode) where filtered messages are saved.
|
35
34
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
defined in self.message_type.
|
35
|
+
Yields:
|
36
|
+
MarketMessage:
|
37
|
+
The next parsed MarketMessage object from the file.
|
40
38
|
|
41
39
|
Raises:
|
42
|
-
ValueError:
|
43
|
-
If a message does not start with the expected 0x00 byte, indicating
|
40
|
+
ValueError:
|
41
|
+
If a message does not start with the expected 0x00 byte, indicating
|
44
42
|
an unexpected file format or possible corruption.
|
45
43
|
|
46
|
-
|
44
|
+
Notes:
|
47
45
|
- Each message starts with a 0x00 byte.
|
48
46
|
- The following byte specifies the message length.
|
49
47
|
- The complete message consists of the first 2 bytes and 'message length' bytes of body.
|
50
|
-
- If a system message (message_type == b'S') with event_code == b'C' is encountered,
|
51
|
-
|
48
|
+
- If a system message (message_type == b'S') with event_code == b'C' is encountered,
|
49
|
+
parsing stops immediately.
|
52
50
|
|
53
51
|
Example:
|
54
|
-
>>>
|
55
|
-
>>>
|
56
|
-
>>>
|
52
|
+
>>> data_file = "01302020.NASDAQ_ITCH50.gz"
|
53
|
+
>>> message_type = b"AFE" # Example message type to filter
|
54
|
+
>>> parser = MessageParser(message_type=message_type)
|
55
|
+
>>> with gzip.open(data_file, "rb") as itch_file:
|
56
|
+
>>> message_count = 0
|
57
|
+
>>> start_time = time.time()
|
58
|
+
>>> for message in parser.read_message_from_file(itch_file):
|
59
|
+
>>> message_count += 1
|
60
|
+
>>> if message_count <= 5:
|
57
61
|
>>> print(message)
|
58
|
-
|
59
|
-
|
60
|
-
|
62
|
+
>>> end_time = time.time()
|
63
|
+
>>> print(f"Processed {message_count} messages in {end_time - start_time:.2f} seconds")
|
64
|
+
>>> print(f"Average time per message: {(end_time - start_time) / message_count:.6f} seconds")
|
65
|
+
"""
|
66
|
+
if not file.readable():
|
67
|
+
raise ValueError("file must be opened in binary read mode")
|
61
68
|
|
62
|
-
|
63
|
-
|
64
|
-
|
69
|
+
if save_file is not None:
|
70
|
+
if not save_file.writable():
|
71
|
+
raise ValueError("save_file must be opened in binary write mode")
|
65
72
|
|
66
|
-
|
67
|
-
|
73
|
+
data_buffer = b""
|
74
|
+
offset = 0
|
75
|
+
|
76
|
+
while True:
|
77
|
+
if len(data_buffer) - offset < 2:
|
78
|
+
data_buffer = data_buffer[offset:]
|
79
|
+
offset = 0
|
68
80
|
new_data = file.read(cachesize)
|
69
81
|
if not new_data:
|
70
82
|
break
|
71
83
|
data_buffer += new_data
|
72
|
-
buffer_len = len(data_buffer)
|
73
|
-
continue
|
74
84
|
|
75
|
-
|
85
|
+
if len(data_buffer) < 2:
|
86
|
+
break
|
87
|
+
|
88
|
+
if data_buffer[offset : offset + 1] != b"\x00":
|
76
89
|
raise ValueError(
|
77
|
-
"Unexpected byte: "
|
90
|
+
"Unexpected byte: "
|
91
|
+
+ str(data_buffer[offset : offset + 1], encoding="ascii")
|
78
92
|
)
|
79
93
|
|
80
|
-
message_len = data_buffer[1]
|
94
|
+
message_len = data_buffer[offset + 1]
|
81
95
|
total_len = 2 + message_len
|
82
96
|
|
83
|
-
if
|
84
|
-
|
97
|
+
if len(data_buffer) - offset < total_len:
|
98
|
+
data_buffer = data_buffer[offset:]
|
99
|
+
offset = 0
|
100
|
+
|
85
101
|
new_data = file.read(cachesize)
|
86
102
|
if not new_data:
|
87
103
|
break
|
88
104
|
data_buffer += new_data
|
89
|
-
buffer_len = len(data_buffer)
|
90
105
|
continue
|
91
|
-
|
106
|
+
|
107
|
+
message_data = data_buffer[offset + 2 : offset + total_len]
|
92
108
|
message = self.get_message_type(message_data)
|
93
109
|
|
94
110
|
if message.message_type in self.message_type:
|
95
|
-
|
111
|
+
if save_file is not None:
|
112
|
+
msg_len_to_bytes = message.message_size.to_bytes()
|
113
|
+
save_file.write(b"\x00" + msg_len_to_bytes + message.to_bytes())
|
114
|
+
yield message
|
96
115
|
|
97
116
|
if message.message_type == b"S": # System message
|
98
117
|
if message.event_code == b"C": # End of messages
|
99
118
|
break
|
119
|
+
offset += total_len
|
100
120
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
if buffer_len < max_message_size and not file_end_reached:
|
106
|
-
new_data = file.read(cachesize)
|
107
|
-
if not new_data:
|
108
|
-
file_end_reached = True
|
109
|
-
else:
|
110
|
-
data_buffer += new_data
|
111
|
-
buffer_len = len(data_buffer)
|
112
|
-
|
113
|
-
return messages
|
114
|
-
|
115
|
-
def read_message_from_bytes(self, data: bytes):
|
121
|
+
def read_message_from_bytes(
|
122
|
+
self, data: bytes, save_file: IO = None
|
123
|
+
) -> Iterator[MarketMessage]:
|
116
124
|
"""
|
117
125
|
Process one or multiple ITCH binary messages from a raw bytes input.
|
118
126
|
|
119
127
|
Args:
|
120
128
|
data (bytes): Binary blob containing one or more ITCH messages.
|
121
129
|
|
122
|
-
|
123
|
-
|
130
|
+
save_file (IO, optional):
|
131
|
+
A binary file-like object (opened in binary write mode) where filtered messages are saved.
|
132
|
+
|
133
|
+
Yields:
|
134
|
+
MarketMessage:
|
135
|
+
The next parsed MarketMessage object from the bytes input.
|
124
136
|
|
125
137
|
Notes:
|
126
138
|
- Each message must be prefixed with a 0x00 header and a length byte.
|
127
139
|
- No buffering is done here — this is meant for real-time decoding.
|
128
140
|
"""
|
141
|
+
if not isinstance(data, (bytes, bytearray)):
|
142
|
+
raise TypeError("data must be bytes or bytearray not " + str(type(data)))
|
143
|
+
|
144
|
+
if save_file is not None:
|
145
|
+
if not save_file.writable():
|
146
|
+
raise ValueError("save_file must be opened in binary write mode")
|
129
147
|
|
130
148
|
offset = 0
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
149
|
+
data_view = memoryview(data)
|
150
|
+
data_len = len(data_view)
|
151
|
+
|
152
|
+
while offset + 2 <= data_len:
|
153
|
+
if data_view[offset : offset + 1] != b"\x00":
|
135
154
|
raise ValueError(
|
136
|
-
f"Unexpected start byte at offset {offset}: "
|
137
|
-
f"{
|
155
|
+
f"Unexpected start byte at offset {offset:offset+1}: "
|
156
|
+
f"{data_view[offset : offset + 1].tobytes()}"
|
138
157
|
)
|
139
|
-
|
140
|
-
msg_len = data[offset + 1]
|
158
|
+
msg_len = data_view[offset + 1]
|
141
159
|
total_len = 2 + msg_len
|
142
160
|
|
143
|
-
if offset + total_len >
|
161
|
+
if offset + total_len > data_len:
|
144
162
|
break
|
145
163
|
|
146
|
-
raw_msg =
|
147
|
-
message = self.get_message_type(raw_msg)
|
164
|
+
raw_msg = data_view[offset + 2 : offset + total_len]
|
165
|
+
message = self.get_message_type(raw_msg.tobytes())
|
148
166
|
|
149
167
|
if message.message_type in self.message_type:
|
150
|
-
|
168
|
+
if save_file is not None:
|
169
|
+
msg_len_to_bytes = message.message_size.to_bytes()
|
170
|
+
save_file.write(b"\x00" + msg_len_to_bytes + message.to_bytes())
|
171
|
+
yield message
|
151
172
|
|
152
173
|
if message.message_type == b"S": # System message
|
153
174
|
if message.event_code == b"C": # End of messages
|
@@ -155,9 +176,7 @@ class MessageParser(object):
|
|
155
176
|
|
156
177
|
offset += total_len
|
157
178
|
|
158
|
-
|
159
|
-
|
160
|
-
def get_message_type(self, message: bytes) -> Type[MarketMessage]:
|
179
|
+
def get_message_type(self, message: bytes) -> MarketMessage:
|
161
180
|
"""
|
162
181
|
Take an entire bytearray and return the appropriate ITCH message
|
163
182
|
instance based on the message type indicator (first byte of the message).
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: itchfeed
|
3
|
-
Version: 1.0.
|
3
|
+
Version: 1.0.5
|
4
4
|
Summary: Simple parser for ITCH messages
|
5
5
|
Home-page: https://github.com/bbalouki/itch
|
6
6
|
Download-URL: https://pypi.org/project/itchfeed/
|
@@ -52,12 +52,14 @@ Dynamic: summary
|
|
52
52
|
* [Usage](#usage)
|
53
53
|
* [Parsing from a Binary File](#parsing-from-a-binary-file)
|
54
54
|
* [Parsing from Raw Bytes](#parsing-from-raw-bytes)
|
55
|
+
* [Creating Messages Programmatically](#creating-messages-programmatically)
|
56
|
+
* [Example with Real-World Sample Data](#example-with-real-world-sample-data)
|
55
57
|
* [Interpreting Market Data (Conceptual Overview)](#interpreting-market-data-conceptual-overview)
|
56
58
|
* [Supported Message Types](#supported-message-types)
|
57
59
|
* [Data Representation](#data-representation)
|
58
60
|
* [Common Attributes of `MarketMessage`](#common-attributes-of-marketmessage)
|
59
61
|
* [Common Methods of `MarketMessage`](#common-methods-of-marketmessage)
|
60
|
-
* [Serializing Messages with `
|
62
|
+
* [Serializing Messages with `to_bytes()`](#serializing-messages-with-to_bytes)
|
61
63
|
* [Data Types in Parsed Messages](#data-types-in-parsed-messages)
|
62
64
|
* [Error Handling](#error-handling)
|
63
65
|
* [Handling Strategies](#handling-strategies)
|
@@ -168,8 +170,8 @@ try:
|
|
168
170
|
|
169
171
|
except FileNotFoundError:
|
170
172
|
print(f"Error: File not found at {itch_file_path}")
|
171
|
-
except
|
172
|
-
print(f"An error occurred: {e}")
|
173
|
+
except ValueError as e:
|
174
|
+
print(f"An error occurred during parsing: {e}")
|
173
175
|
|
174
176
|
```
|
175
177
|
|
@@ -211,6 +213,78 @@ while not message_queue.empty():
|
|
211
213
|
|
212
214
|
```
|
213
215
|
|
216
|
+
### Creating Messages Programmatically
|
217
|
+
|
218
|
+
In addition to parsing, `itch` provides a simple way to create ITCH message objects from scratch. This is particularly useful for:
|
219
|
+
- **Testing:** Generating specific message sequences to test your own downstream applications.
|
220
|
+
- **Simulation:** Building custom market simulators that produce ITCH-compliant data streams.
|
221
|
+
- **Data Generation:** Creating custom datasets for analysis or backtesting.
|
222
|
+
|
223
|
+
The `create_message` function is the primary tool for this purpose. It takes a `message_type` and keyword arguments corresponding to the desired message attributes.
|
224
|
+
|
225
|
+
Here's a basic example of how to create a `SystemEventMessage` to signal the "Start of Messages":
|
226
|
+
|
227
|
+
```python
|
228
|
+
from itch.messages import create_message, SystemEventMessage
|
229
|
+
|
230
|
+
# Define the attributes for the message
|
231
|
+
event_attributes = {
|
232
|
+
"stock_locate": 1,
|
233
|
+
"tracking_number": 2,
|
234
|
+
"timestamp": 1651500000 * 1_000_000_000,
|
235
|
+
"event_code": b"O"
|
236
|
+
}
|
237
|
+
|
238
|
+
# Create the message object
|
239
|
+
system_event_message = create_message(b"S", **event_attributes)
|
240
|
+
|
241
|
+
# You can now work with this object just like one from the parser
|
242
|
+
print(isinstance(system_event_message, SystemEventMessage))
|
243
|
+
# Expected output: True
|
244
|
+
|
245
|
+
print(system_event_message)
|
246
|
+
# Expected output: SystemEventMessage(description='System Event Message', event_code='O', message_format='!HHHIc', message_pack_format='!cHHHIc', message_size=12, message_type='S', price_precision=4, stock_locate=1, timestamp=86311638581248, tracking_number=2)
|
247
|
+
```
|
248
|
+
|
249
|
+
### Example with Real-World Sample Data
|
250
|
+
|
251
|
+
You can also use the sample data provided in `tests/data.py` to create messages, simulating a more realistic scenario.
|
252
|
+
|
253
|
+
```python
|
254
|
+
from itch.messages import create_message, AddOrderNoMPIAttributionMessage
|
255
|
+
from tests.data import TEST_DATA
|
256
|
+
|
257
|
+
# Get the sample data for an "Add Order" message (type 'A')
|
258
|
+
add_order_data = TEST_DATA[b"A"]
|
259
|
+
|
260
|
+
# Create the message
|
261
|
+
add_order_message = create_message(b"A", **add_order_data)
|
262
|
+
|
263
|
+
# Verify the type
|
264
|
+
print(isinstance(add_order_message, AddOrderNoMPIAttributionMessage))
|
265
|
+
# Expected output: True
|
266
|
+
|
267
|
+
# Access its attributes
|
268
|
+
print(f"Stock: {add_order_message.stock.decode().strip()}")
|
269
|
+
# Expected output: Stock: AAPL
|
270
|
+
|
271
|
+
print(f"Price: {add_order_message.decode_price('price')}")
|
272
|
+
# Expected output: Price: 150.1234
|
273
|
+
|
274
|
+
# Test all message types in the sample data
|
275
|
+
for message_type, sample_data in TEST_DATA.items():
|
276
|
+
print(f"Creating message of type {message_type}")
|
277
|
+
message = create_message(message_type, **sample_data)
|
278
|
+
print(f"Created message: {message}")
|
279
|
+
print(f"Packed message: {message.to_bytes()}")
|
280
|
+
print(f"Message size: {message.message_size}")
|
281
|
+
print(f"Message Attributes: {message.get_attributes()}")
|
282
|
+
assert len(message.to_bytes()) == message.message_size
|
283
|
+
print()
|
284
|
+
```
|
285
|
+
|
286
|
+
By leveraging `create_message`, you can build robust test suites for your trading algorithms, compliance checks, or data analysis pipelines without needing a live data feed.
|
287
|
+
|
214
288
|
## Interpreting Market Data (Conceptual Overview)
|
215
289
|
|
216
290
|
Parsing individual ITCH messages is the first step; understanding market dynamics often requires processing and correlating a sequence of these messages. This library provides the tools to decode messages, but interpreting their collective meaning requires building further logic.
|
@@ -314,14 +388,14 @@ The `MarketMessage` base class, and therefore all specific message classes, prov
|
|
314
388
|
* Returns a dictionary of all attributes (fields) of the message instance, along with their current values.
|
315
389
|
* This can be useful for generic inspection or logging of message contents without needing to know the specific type of the message beforehand.
|
316
390
|
|
317
|
-
### Serializing Messages with `
|
391
|
+
### Serializing Messages with `to_bytes()`
|
318
392
|
|
319
|
-
Each specific message class (e.g., `SystemEventMessage`, `AddOrderNoMPIAttributionMessage`) also provides a `
|
393
|
+
Each specific message class (e.g., `SystemEventMessage`, `AddOrderNoMPIAttributionMessage`) also provides a `to_bytes()` method. This method is the inverse of the parsing process.
|
320
394
|
|
321
395
|
* **Purpose:** It serializes the message object, with its current attribute values, back into its raw ITCH 5.0 binary format. The output is a `bytes` object representing the exact byte sequence that would appear in an ITCH data feed for that message.
|
322
396
|
* **Usefulness:**
|
323
397
|
* **Generating Test Data:** Create custom ITCH messages for testing your own ITCH processing applications.
|
324
|
-
* **Modifying Messages:** Parse an existing message, modify some of its attributes, and then `
|
398
|
+
* **Modifying Messages:** Parse an existing message, modify some of its attributes, and then `to_bytes()` it back into binary form.
|
325
399
|
* **Creating Custom ITCH Feeds:** While more involved, you could use this to construct sequences of ITCH messages for specialized scenarios.
|
326
400
|
|
327
401
|
**Example:**
|
@@ -341,21 +415,21 @@ import time
|
|
341
415
|
# for packing requires setting attributes manually if not using raw bytes for construction)
|
342
416
|
|
343
417
|
event_msg = SystemEventMessage.__new__(SystemEventMessage) # Create instance without calling __init__
|
344
|
-
event_msg.message_type = b'S' # Must be set for
|
418
|
+
event_msg.message_type = b'S' # Must be set for to_bytes() to know its type
|
345
419
|
event_msg.stock_locate = 0 # Placeholder or actual value
|
346
420
|
event_msg.tracking_number = 0 # Placeholder or actual value
|
347
421
|
event_msg.event_code = b'O' # Example: Start of Messages
|
348
422
|
|
349
423
|
# 2. Set the timestamp.
|
350
424
|
# The `timestamp` attribute (nanoseconds since midnight) must be set.
|
351
|
-
# The `
|
425
|
+
# The `to_bytes()` method will internally use `split_timestamp()` to get the parts.
|
352
426
|
current_nanoseconds = int(time.time() * 1e9) % (24 * 60 * 60 * int(1e9))
|
353
427
|
event_msg.timestamp = current_nanoseconds # Directly set the nanosecond timestamp
|
354
428
|
|
355
429
|
# 3. Pack the message into binary format.
|
356
|
-
# The
|
430
|
+
# The to_bytes() method prepends the message type and then packs stock_locate,
|
357
431
|
# tracking_number, the split timestamp, and then the message-specific fields.
|
358
|
-
packed_bytes = event_msg.
|
432
|
+
packed_bytes = event_msg.to_bytes()
|
359
433
|
|
360
434
|
# 4. The result is a bytes object
|
361
435
|
print(f"Packed {len(packed_bytes)} bytes: {packed_bytes.hex().upper()}")
|
@@ -0,0 +1,9 @@
|
|
1
|
+
itch/__init__.py,sha256=M9Jirj4-XXdaCoTcU2_g89z7JHK8mDtJflTFf9HnU-k,205
|
2
|
+
itch/indicators.py,sha256=-Ed2M8I60xGQ1bIPZCGCKGb8ayT87JAnIaosfiBimXI,6542
|
3
|
+
itch/messages.py,sha256=UWhpFzgfaDwAah-R0p24LtpP7-G5pP1jnOMakBc33t4,64935
|
4
|
+
itch/parser.py,sha256=glYjyeqY9841UqcOD8Wck-Az70AUwQxTt9SckYoxOg0,7422
|
5
|
+
itchfeed-1.0.5.dist-info/licenses/LICENSE,sha256=f2u79rUzh-UcYH0RN0Ph0VvVYHBkYlVxtguhKmrHqsw,1089
|
6
|
+
itchfeed-1.0.5.dist-info/METADATA,sha256=gUXQ25Q5Cvw6EHGVLUDuTXFOTdUSp8VQEIyKUOd8LU8,33853
|
7
|
+
itchfeed-1.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
8
|
+
itchfeed-1.0.5.dist-info/top_level.txt,sha256=xwsOYShvy3gc1rfyitCTgSxBZDGG1y6bfQxkdhIGmEM,5
|
9
|
+
itchfeed-1.0.5.dist-info/RECORD,,
|
itchfeed-1.0.3.dist-info/RECORD
DELETED
@@ -1,9 +0,0 @@
|
|
1
|
-
itch/__init__.py,sha256=m8ulOH0nET4yLoKcqWJA-VjOEZldGo932t95rHOZh4w,204
|
2
|
-
itch/indicators.py,sha256=-Ed2M8I60xGQ1bIPZCGCKGb8ayT87JAnIaosfiBimXI,6542
|
3
|
-
itch/messages.py,sha256=UHr4eBTtgiLg9vO53GpciAxiAO9AD9uCu8Xz3WnfadM,61927
|
4
|
-
itch/parser.py,sha256=BOrkGsmRkcYnXSf5S9yqrwzP0jqtkH5mGCpWCeiWNTg,6155
|
5
|
-
itchfeed-1.0.3.dist-info/licenses/LICENSE,sha256=f2u79rUzh-UcYH0RN0Ph0VvVYHBkYlVxtguhKmrHqsw,1089
|
6
|
-
itchfeed-1.0.3.dist-info/METADATA,sha256=nfZbTBXUq5Ghq1xhls4fttblPNlpRGtqZth1h8lnx-4,30699
|
7
|
-
itchfeed-1.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
8
|
-
itchfeed-1.0.3.dist-info/top_level.txt,sha256=xwsOYShvy3gc1rfyitCTgSxBZDGG1y6bfQxkdhIGmEM,5
|
9
|
-
itchfeed-1.0.3.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|