nuro-vtf 1.0.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. nuro_vtf-1.0.6/LICENSE +21 -0
  2. nuro_vtf-1.0.6/PKG-INFO +192 -0
  3. nuro_vtf-1.0.6/README.md +158 -0
  4. nuro_vtf-1.0.6/nuro_vtf.egg-info/PKG-INFO +192 -0
  5. nuro_vtf-1.0.6/nuro_vtf.egg-info/SOURCES.txt +12 -0
  6. nuro_vtf-1.0.6/nuro_vtf.egg-info/dependency_links.txt +1 -0
  7. nuro_vtf-1.0.6/nuro_vtf.egg-info/requires.txt +1 -0
  8. nuro_vtf-1.0.6/nuro_vtf.egg-info/top_level.txt +3 -0
  9. nuro_vtf-1.0.6/nuro_vtf.py +9 -0
  10. nuro_vtf-1.0.6/pyproject.toml +61 -0
  11. nuro_vtf-1.0.6/setup.cfg +4 -0
  12. nuro_vtf-1.0.6/setup.py +79 -0
  13. nuro_vtf-1.0.6/vtf.py +781 -0
  14. nuro_vtf-1.0.6/vtf_core.cpp +1700 -0
nuro_vtf-1.0.6/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 venkey2k
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
nuro_vtf-1.0.6/PKG-INFO ADDED
@@ -0,0 +1,192 @@
1
+ Metadata-Version: 2.4
2
+ Name: nuro-vtf
3
+ Version: 1.0.6
4
+ Summary: VTF (Venkey Tick Format) v1.0 — High-Frequency Option Market Data Format & Zero-Copy Engine
5
+ Home-page: https://github.com/venkey2k/VTF
6
+ Author: venkey2k
7
+ Author-email: venkey2k <venkey2kkid@gmail.com>
8
+ License-Expression: MIT
9
+ Project-URL: Homepage, https://github.com/venkey2k/VTF
10
+ Project-URL: Repository, https://github.com/venkey2k/VTF.git
11
+ Project-URL: Issues, https://github.com/venkey2k/VTF/issues
12
+ Keywords: hft,market-data,options,nifty,sensex,orderbook,greeks,binary-format,lz4,simd,zero-copy,vtf
13
+ Classifier: Development Status :: 5 - Production/Stable
14
+ Classifier: Intended Audience :: Financial and Insurance Industry
15
+ Classifier: Topic :: Office/Business :: Financial :: Investment
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Programming Language :: C++
23
+ Classifier: Operating System :: POSIX :: Linux
24
+ Classifier: Operating System :: Microsoft :: Windows
25
+ Classifier: Operating System :: MacOS
26
+ Requires-Python: >=3.9
27
+ Description-Content-Type: text/markdown
28
+ License-File: LICENSE
29
+ Requires-Dist: numpy>=1.20
30
+ Dynamic: author
31
+ Dynamic: home-page
32
+ Dynamic: license-file
33
+ Dynamic: requires-python
34
+
35
+ # VTF (Venkey Tick Format) v1.0
36
+
37
+ [![CI](https://github.com/venkey2k/VTF/actions/workflows/ci.yml/badge.svg)](https://github.com/venkey2k/VTF/actions/workflows/ci.yml)
38
+ [![PyPI version](https://img.shields.io/pypi/v/nuro-vtf.svg)](https://pypi.org/project/nuro-vtf/)
39
+ [![Python versions](https://img.shields.io/pypi/pyversions/nuro-vtf.svg)](https://pypi.org/project/nuro-vtf/)
40
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
41
+
42
+ **VTF (Venkey Tick Format)** is a custom binary market-data storage format and zero-copy streaming engine engineered specifically for high-frequency **NSE (NIFTY 50)** and **BSE (SENSEX)** equity index options.
43
+
44
+ It captures trades, 5-level Market-By-Price (L2) order books, Open Interest (OI), Implied Volatility (IV), and Black-Scholes Greeks ($\Delta, \Gamma, \Theta, \mathcal{V}$) in a 156-byte compact binary tick representation, striking an optimal balance between **ultra-fast write throughput**, **rapid random query speeds**, and **extreme compression**.
45
+
46
+ ---
47
+
48
+ ## ⚡ Performance Highlights (500,000 Option Ticks Benchmark)
49
+
50
+ | Format / Configuration | File Size (MiB) | Bytes / Tick | Compression Ratio | Write Speed (MT/s) | Effective Write (MiB/s) | Read Speed (MT/s) | Mid-Write Crash Resilience |
51
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
52
+ | **VTF v1.0 (`LZ4-Fast`, accel=4)** | **8.496** | **17.82 B** | **8.76x** | **3.54 MT/s** | **526.7 MiB/s** | **2.95 MT/s** | **Yes (Block CRC32C)** |
53
+ | **VTF v1.0 (`Zstd-L3`)** | **5.067** | **10.63 B** | **14.68x** | **2.14 MT/s** | **319.0 MiB/s** | **2.50 MT/s** | **Yes (Block CRC32C)** |
54
+ | Apache Parquet (`Snappy`) | 27.127 | 56.89 B | 2.74x | 0.63 MT/s | 94.3 MiB/s | 3.10 MT/s | **No** (Unreadable if unclosed) |
55
+ | Apache Parquet (`Zstd-L3`) | 23.945 | 50.22 B | 3.11x | 0.53 MT/s | 78.6 MiB/s | 2.80 MT/s | **No** (Unreadable if unclosed) |
56
+ | Arrow IPC / Feather (`LZ4`) | 51.027 | 107.01 B | 1.46x | 1.82 MT/s | 270.1 MiB/s | 3.39 MT/s | **No** (Unreadable if unclosed) |
57
+ | Raw Structs + LZ4 | 30.627 | 64.23 B | 2.43x | 2.21 MT/s | 328.6 MiB/s | 3.81 MT/s | Partial (No index/CRC) |
58
+
59
+ - **5.59x faster writes** and **3.19x smaller files** than Apache Parquet (`Snappy`).
60
+ - **100% crash recovery**: Zero loss of flushed blocks if process is terminated mid-write (`SIGKILL`, power failure).
61
+ - **Direct NumPy zero-copy integration**: Structured array `VTF_TICK_DTYPE` matches C++ memory layout with 0 serialization copies.
62
+
63
+ ---
64
+
65
+ ## 🚀 Installation
66
+
67
+ Install pre-built binary wheels directly from PyPI:
68
+
69
+ ```bash
70
+ pip install nuro-vtf
71
+ ```
72
+
73
+ Or install from source with full CPU-native optimizations:
74
+
75
+ ```bash
76
+ git clone https://github.com/venkey2k/VTF.git
77
+ cd VTF
78
+ pip install -v .
79
+ ```
80
+
81
+ *Compatible with Linux (x86_64, aarch64), Windows (x64), and macOS (Apple Silicon & Intel).*
82
+
83
+ ---
84
+
85
+ ## 💡 Quickstart (Python API)
86
+
87
+ ```python
88
+ import numpy as np
89
+ import vtf
90
+
91
+ # 1. Allocate structured NumPy buffer matching the 156-byte tick schema
92
+ num_ticks = 32_768
93
+ ticks = np.zeros(num_ticks, dtype=vtf.VTF_TICK_DTYPE)
94
+
95
+ ticks["timestamp_ns"] = 1_792_554_300_000_000_000 + np.arange(num_ticks, dtype=np.int64) * 10_000
96
+ ticks["instrument_id"] = 1001 # NIFTY 24500 CE token
97
+ ticks["exchange_id"] = vtf.EXCHANGE_NSE_NIFTY
98
+ ticks["ltp_paise"] = 18505 # 185.05 INR
99
+ ticks["ltq"] = 75
100
+ ticks["volume"] = 150_000
101
+ ticks["oi"] = 1_250_000
102
+ ticks["iv_bps"] = 1645 # 16.45% IV
103
+ ticks["delta_bps"] = 5120 # +0.5120 Delta
104
+
105
+ # 2. Write in self-framing PAX micro-blocks (16,384 ticks/block)
106
+ with vtf.VTFWriter("nifty_options.vtf", codec=vtf.CODEC_LZ4_FAST) as writer:
107
+ writer.append(ticks)
108
+
109
+ # 3. Read back or run pushdown range queries
110
+ with vtf.VTFReader("nifty_options.vtf") as reader:
111
+ print(reader.header_info())
112
+
113
+ # Read full stream
114
+ all_ticks = reader.read_all(verify_crc=True)
115
+
116
+ # Filter by time range and contract token using block-level Bloom filters
117
+ subset = reader.query(
118
+ start_ns=int(ticks["timestamp_ns"][1000]),
119
+ end_ns=int(ticks["timestamp_ns"][5000]),
120
+ instrument_id=1001,
121
+ )
122
+ print(f"Retrieved {len(subset):,} filtered ticks in microseconds.")
123
+
124
+ # 4. Crash Recovery (in-place repair for unfinalized files)
125
+ report = vtf.recover_vtf_file("nifty_options.vtf")
126
+ print(f"Valid blocks preserved: {report.valid_blocks}, rows recovered: {report.valid_rows:,}")
127
+ ```
128
+
129
+ ---
130
+
131
+ ## 🛠️ The VTF Encoding Pipeline
132
+
133
+ ```
134
+ Raw Multiplexed Option Ticks (156 B/row, interleaved strikes)
135
+ │
136
+ ▼
137
+ Stage 1: Per-Instrument State Cache (512-slot hash table)
138
+ Delta-encoded against previous tick of SAME contract
139
+ │
140
+ ▼
141
+ Stage 2: Relational Order Book Ladder Encoding
142
+ Bid[0] vs LTP, Ask[0] spread, L2-L5 depth step diffs
143
+ │
144
+ ▼
145
+ Stage 3: ZigZag Variable-Length Delta Transformation
146
+ Maps small positive/negative jumps to small unsigned integers
147
+ │
148
+ ▼
149
+ Stage 4: SIMD SSSE3 4x4 Byte-Stream-Split (BSS) Transposition
150
+ Clusters upper 0x00 bytes into contiguous byte planes
151
+ │
152
+ ▼
153
+ Stage 5: High-Speed Block Codec (LZ4-Fast accel=4 / Zstd)
154
+ Instant hardware CRC32C verification on flush
155
+ ```
156
+
157
+ ---
158
+
159
+ ## 📦 Repository Structure
160
+
161
+ ```
162
+ VTF/
163
+ ├── vtf.py # Python API & NumPy ctypes wrapper
164
+ ├── vtf.hpp # C/C++ public API header
165
+ ├── vtf_core.cpp # Cross-platform zero-dependency C++17 engine
166
+ ├── benchmark_vtf.py # 500k tick benchmark suite against Parquet & Arrow
167
+ ├── example_quickstart.py # Ready-to-run quickstart example
168
+ ├── VTF_Specification_and_Benchmarks.md # Complete wire format specification
169
+ ├── setup.py # C++ extension setuptools builder
170
+ ├── pyproject.toml # PEP 517 build & cibuildwheel config
171
+ ├── LICENSE # MIT License
172
+ └── .github/workflows/
173
+ ├── ci.yml # Multi-OS CI testing (Linux, Windows, macOS)
174
+ └── release.yml # Automated multi-arch wheel build & PyPI publish
175
+ ```
176
+
177
+ ---
178
+
179
+ ## 🔬 Benchmark Verification
180
+
181
+ To run the complete benchmark suite comparing VTF with Parquet and Arrow:
182
+
183
+ ```bash
184
+ pip install numpy pyarrow
185
+ python benchmark_vtf.py
186
+ ```
187
+
188
+ ---
189
+
190
+ ## 📄 License
191
+
192
+ VTF is released under the [MIT License](LICENSE). Copyright (c) 2026 venkey2k.
nuro_vtf-1.0.6/README.md ADDED
@@ -0,0 +1,158 @@
1
+ # VTF (Venkey Tick Format) v1.0
2
+
3
+ [![CI](https://github.com/venkey2k/VTF/actions/workflows/ci.yml/badge.svg)](https://github.com/venkey2k/VTF/actions/workflows/ci.yml)
4
+ [![PyPI version](https://img.shields.io/pypi/v/nuro-vtf.svg)](https://pypi.org/project/nuro-vtf/)
5
+ [![Python versions](https://img.shields.io/pypi/pyversions/nuro-vtf.svg)](https://pypi.org/project/nuro-vtf/)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
7
+
8
+ **VTF (Venkey Tick Format)** is a custom binary market-data storage format and zero-copy streaming engine engineered specifically for high-frequency **NSE (NIFTY 50)** and **BSE (SENSEX)** equity index options.
9
+
10
+ It captures trades, 5-level Market-By-Price (L2) order books, Open Interest (OI), Implied Volatility (IV), and Black-Scholes Greeks ($\Delta, \Gamma, \Theta, \mathcal{V}$) in a 156-byte compact binary tick representation, striking an optimal balance between **ultra-fast write throughput**, **rapid random query speeds**, and **extreme compression**.
11
+
12
+ ---
13
+
14
+ ## ⚡ Performance Highlights (500,000 Option Ticks Benchmark)
15
+
16
+ | Format / Configuration | File Size (MiB) | Bytes / Tick | Compression Ratio | Write Speed (MT/s) | Effective Write (MiB/s) | Read Speed (MT/s) | Mid-Write Crash Resilience |
17
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
18
+ | **VTF v1.0 (`LZ4-Fast`, accel=4)** | **8.496** | **17.82 B** | **8.76x** | **3.54 MT/s** | **526.7 MiB/s** | **2.95 MT/s** | **Yes (Block CRC32C)** |
19
+ | **VTF v1.0 (`Zstd-L3`)** | **5.067** | **10.63 B** | **14.68x** | **2.14 MT/s** | **319.0 MiB/s** | **2.50 MT/s** | **Yes (Block CRC32C)** |
20
+ | Apache Parquet (`Snappy`) | 27.127 | 56.89 B | 2.74x | 0.63 MT/s | 94.3 MiB/s | 3.10 MT/s | **No** (Unreadable if unclosed) |
21
+ | Apache Parquet (`Zstd-L3`) | 23.945 | 50.22 B | 3.11x | 0.53 MT/s | 78.6 MiB/s | 2.80 MT/s | **No** (Unreadable if unclosed) |
22
+ | Arrow IPC / Feather (`LZ4`) | 51.027 | 107.01 B | 1.46x | 1.82 MT/s | 270.1 MiB/s | 3.39 MT/s | **No** (Unreadable if unclosed) |
23
+ | Raw Structs + LZ4 | 30.627 | 64.23 B | 2.43x | 2.21 MT/s | 328.6 MiB/s | 3.81 MT/s | Partial (No index/CRC) |
24
+
25
+ - **5.59x faster writes** and **3.19x smaller files** than Apache Parquet (`Snappy`).
26
+ - **100% crash recovery**: Zero loss of flushed blocks if process is terminated mid-write (`SIGKILL`, power failure).
27
+ - **Direct NumPy zero-copy integration**: Structured array `VTF_TICK_DTYPE` matches C++ memory layout with 0 serialization copies.
28
+
29
+ ---
30
+
31
+ ## 🚀 Installation
32
+
33
+ Install pre-built binary wheels directly from PyPI:
34
+
35
+ ```bash
36
+ pip install nuro-vtf
37
+ ```
38
+
39
+ Or install from source with full CPU-native optimizations:
40
+
41
+ ```bash
42
+ git clone https://github.com/venkey2k/VTF.git
43
+ cd VTF
44
+ pip install -v .
45
+ ```
46
+
47
+ *Compatible with Linux (x86_64, aarch64), Windows (x64), and macOS (Apple Silicon & Intel).*
48
+
49
+ ---
50
+
51
+ ## 💡 Quickstart (Python API)
52
+
53
+ ```python
54
+ import numpy as np
55
+ import vtf
56
+
57
+ # 1. Allocate structured NumPy buffer matching the 156-byte tick schema
58
+ num_ticks = 32_768
59
+ ticks = np.zeros(num_ticks, dtype=vtf.VTF_TICK_DTYPE)
60
+
61
+ ticks["timestamp_ns"] = 1_792_554_300_000_000_000 + np.arange(num_ticks, dtype=np.int64) * 10_000
62
+ ticks["instrument_id"] = 1001 # NIFTY 24500 CE token
63
+ ticks["exchange_id"] = vtf.EXCHANGE_NSE_NIFTY
64
+ ticks["ltp_paise"] = 18505 # 185.05 INR
65
+ ticks["ltq"] = 75
66
+ ticks["volume"] = 150_000
67
+ ticks["oi"] = 1_250_000
68
+ ticks["iv_bps"] = 1645 # 16.45% IV
69
+ ticks["delta_bps"] = 5120 # +0.5120 Delta
70
+
71
+ # 2. Write in self-framing PAX micro-blocks (16,384 ticks/block)
72
+ with vtf.VTFWriter("nifty_options.vtf", codec=vtf.CODEC_LZ4_FAST) as writer:
73
+ writer.append(ticks)
74
+
75
+ # 3. Read back or run pushdown range queries
76
+ with vtf.VTFReader("nifty_options.vtf") as reader:
77
+ print(reader.header_info())
78
+
79
+ # Read full stream
80
+ all_ticks = reader.read_all(verify_crc=True)
81
+
82
+ # Filter by time range and contract token using block-level Bloom filters
83
+ subset = reader.query(
84
+ start_ns=int(ticks["timestamp_ns"][1000]),
85
+ end_ns=int(ticks["timestamp_ns"][5000]),
86
+ instrument_id=1001,
87
+ )
88
+ print(f"Retrieved {len(subset):,} filtered ticks in microseconds.")
89
+
90
+ # 4. Crash Recovery (in-place repair for unfinalized files)
91
+ report = vtf.recover_vtf_file("nifty_options.vtf")
92
+ print(f"Valid blocks preserved: {report.valid_blocks}, rows recovered: {report.valid_rows:,}")
93
+ ```
94
+
95
+ ---
96
+
97
+ ## 🛠️ The VTF Encoding Pipeline
98
+
99
+ ```
100
+ Raw Multiplexed Option Ticks (156 B/row, interleaved strikes)
101
+ │
102
+ ▼
103
+ Stage 1: Per-Instrument State Cache (512-slot hash table)
104
+ Delta-encoded against previous tick of SAME contract
105
+ │
106
+ ▼
107
+ Stage 2: Relational Order Book Ladder Encoding
108
+ Bid[0] vs LTP, Ask[0] spread, L2-L5 depth step diffs
109
+ │
110
+ ▼
111
+ Stage 3: ZigZag Variable-Length Delta Transformation
112
+ Maps small positive/negative jumps to small unsigned integers
113
+ │
114
+ ▼
115
+ Stage 4: SIMD SSSE3 4x4 Byte-Stream-Split (BSS) Transposition
116
+ Clusters upper 0x00 bytes into contiguous byte planes
117
+ │
118
+ ▼
119
+ Stage 5: High-Speed Block Codec (LZ4-Fast accel=4 / Zstd)
120
+ Instant hardware CRC32C verification on flush
121
+ ```
122
+
123
+ ---
124
+
125
+ ## 📦 Repository Structure
126
+
127
+ ```
128
+ VTF/
129
+ ├── vtf.py # Python API & NumPy ctypes wrapper
130
+ ├── vtf.hpp # C/C++ public API header
131
+ ├── vtf_core.cpp # Cross-platform zero-dependency C++17 engine
132
+ ├── benchmark_vtf.py # 500k tick benchmark suite against Parquet & Arrow
133
+ ├── example_quickstart.py # Ready-to-run quickstart example
134
+ ├── VTF_Specification_and_Benchmarks.md # Complete wire format specification
135
+ ├── setup.py # C++ extension setuptools builder
136
+ ├── pyproject.toml # PEP 517 build & cibuildwheel config
137
+ ├── LICENSE # MIT License
138
+ └── .github/workflows/
139
+ ├── ci.yml # Multi-OS CI testing (Linux, Windows, macOS)
140
+ └── release.yml # Automated multi-arch wheel build & PyPI publish
141
+ ```
142
+
143
+ ---
144
+
145
+ ## 🔬 Benchmark Verification
146
+
147
+ To run the complete benchmark suite comparing VTF with Parquet and Arrow:
148
+
149
+ ```bash
150
+ pip install numpy pyarrow
151
+ python benchmark_vtf.py
152
+ ```
153
+
154
+ ---
155
+
156
+ ## 📄 License
157
+
158
+ VTF is released under the [MIT License](LICENSE). Copyright (c) 2026 venkey2k.
nuro_vtf-1.0.6/nuro_vtf.egg-info/PKG-INFO ADDED
@@ -0,0 +1,192 @@
1
+ Metadata-Version: 2.4
2
+ Name: nuro-vtf
3
+ Version: 1.0.6
4
+ Summary: VTF (Venkey Tick Format) v1.0 — High-Frequency Option Market Data Format & Zero-Copy Engine
5
+ Home-page: https://github.com/venkey2k/VTF
6
+ Author: venkey2k
7
+ Author-email: venkey2k <venkey2kkid@gmail.com>
8
+ License-Expression: MIT
9
+ Project-URL: Homepage, https://github.com/venkey2k/VTF
10
+ Project-URL: Repository, https://github.com/venkey2k/VTF.git
11
+ Project-URL: Issues, https://github.com/venkey2k/VTF/issues
12
+ Keywords: hft,market-data,options,nifty,sensex,orderbook,greeks,binary-format,lz4,simd,zero-copy,vtf
13
+ Classifier: Development Status :: 5 - Production/Stable
14
+ Classifier: Intended Audience :: Financial and Insurance Industry
15
+ Classifier: Topic :: Office/Business :: Financial :: Investment
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Programming Language :: C++
23
+ Classifier: Operating System :: POSIX :: Linux
24
+ Classifier: Operating System :: Microsoft :: Windows
25
+ Classifier: Operating System :: MacOS
26
+ Requires-Python: >=3.9
27
+ Description-Content-Type: text/markdown
28
+ License-File: LICENSE
29
+ Requires-Dist: numpy>=1.20
30
+ Dynamic: author
31
+ Dynamic: home-page
32
+ Dynamic: license-file
33
+ Dynamic: requires-python
34
+
35
+ # VTF (Venkey Tick Format) v1.0
36
+
37
+ [![CI](https://github.com/venkey2k/VTF/actions/workflows/ci.yml/badge.svg)](https://github.com/venkey2k/VTF/actions/workflows/ci.yml)
38
+ [![PyPI version](https://img.shields.io/pypi/v/nuro-vtf.svg)](https://pypi.org/project/nuro-vtf/)
39
+ [![Python versions](https://img.shields.io/pypi/pyversions/nuro-vtf.svg)](https://pypi.org/project/nuro-vtf/)
40
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
41
+
42
+ **VTF (Venkey Tick Format)** is a custom binary market-data storage format and zero-copy streaming engine engineered specifically for high-frequency **NSE (NIFTY 50)** and **BSE (SENSEX)** equity index options.
43
+
44
+ It captures trades, 5-level Market-By-Price (L2) order books, Open Interest (OI), Implied Volatility (IV), and Black-Scholes Greeks ($\Delta, \Gamma, \Theta, \mathcal{V}$) in a 156-byte compact binary tick representation, striking an optimal balance between **ultra-fast write throughput**, **rapid random query speeds**, and **extreme compression**.
45
+
46
+ ---
47
+
48
+ ## ⚡ Performance Highlights (500,000 Option Ticks Benchmark)
49
+
50
+ | Format / Configuration | File Size (MiB) | Bytes / Tick | Compression Ratio | Write Speed (MT/s) | Effective Write (MiB/s) | Read Speed (MT/s) | Mid-Write Crash Resilience |
51
+ | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |
52
+ | **VTF v1.0 (`LZ4-Fast`, accel=4)** | **8.496** | **17.82 B** | **8.76x** | **3.54 MT/s** | **526.7 MiB/s** | **2.95 MT/s** | **Yes (Block CRC32C)** |
53
+ | **VTF v1.0 (`Zstd-L3`)** | **5.067** | **10.63 B** | **14.68x** | **2.14 MT/s** | **319.0 MiB/s** | **2.50 MT/s** | **Yes (Block CRC32C)** |
54
+ | Apache Parquet (`Snappy`) | 27.127 | 56.89 B | 2.74x | 0.63 MT/s | 94.3 MiB/s | 3.10 MT/s | **No** (Unreadable if unclosed) |
55
+ | Apache Parquet (`Zstd-L3`) | 23.945 | 50.22 B | 3.11x | 0.53 MT/s | 78.6 MiB/s | 2.80 MT/s | **No** (Unreadable if unclosed) |
56
+ | Arrow IPC / Feather (`LZ4`) | 51.027 | 107.01 B | 1.46x | 1.82 MT/s | 270.1 MiB/s | 3.39 MT/s | **No** (Unreadable if unclosed) |
57
+ | Raw Structs + LZ4 | 30.627 | 64.23 B | 2.43x | 2.21 MT/s | 328.6 MiB/s | 3.81 MT/s | Partial (No index/CRC) |
58
+
59
+ - **5.59x faster writes** and **3.19x smaller files** than Apache Parquet (`Snappy`).
60
+ - **100% crash recovery**: Zero loss of flushed blocks if process is terminated mid-write (`SIGKILL`, power failure).
61
+ - **Direct NumPy zero-copy integration**: Structured array `VTF_TICK_DTYPE` matches C++ memory layout with 0 serialization copies.
62
+
63
+ ---
64
+
65
+ ## 🚀 Installation
66
+
67
+ Install pre-built binary wheels directly from PyPI:
68
+
69
+ ```bash
70
+ pip install nuro-vtf
71
+ ```
72
+
73
+ Or install from source with full CPU-native optimizations:
74
+
75
+ ```bash
76
+ git clone https://github.com/venkey2k/VTF.git
77
+ cd VTF
78
+ pip install -v .
79
+ ```
80
+
81
+ *Compatible with Linux (x86_64, aarch64), Windows (x64), and macOS (Apple Silicon & Intel).*
82
+
83
+ ---
84
+
85
+ ## 💡 Quickstart (Python API)
86
+
87
+ ```python
88
+ import numpy as np
89
+ import vtf
90
+
91
+ # 1. Allocate structured NumPy buffer matching the 156-byte tick schema
92
+ num_ticks = 32_768
93
+ ticks = np.zeros(num_ticks, dtype=vtf.VTF_TICK_DTYPE)
94
+
95
+ ticks["timestamp_ns"] = 1_792_554_300_000_000_000 + np.arange(num_ticks, dtype=np.int64) * 10_000
96
+ ticks["instrument_id"] = 1001 # NIFTY 24500 CE token
97
+ ticks["exchange_id"] = vtf.EXCHANGE_NSE_NIFTY
98
+ ticks["ltp_paise"] = 18505 # 185.05 INR
99
+ ticks["ltq"] = 75
100
+ ticks["volume"] = 150_000
101
+ ticks["oi"] = 1_250_000
102
+ ticks["iv_bps"] = 1645 # 16.45% IV
103
+ ticks["delta_bps"] = 5120 # +0.5120 Delta
104
+
105
+ # 2. Write in self-framing PAX micro-blocks (16,384 ticks/block)
106
+ with vtf.VTFWriter("nifty_options.vtf", codec=vtf.CODEC_LZ4_FAST) as writer:
107
+ writer.append(ticks)
108
+
109
+ # 3. Read back or run pushdown range queries
110
+ with vtf.VTFReader("nifty_options.vtf") as reader:
111
+ print(reader.header_info())
112
+
113
+ # Read full stream
114
+ all_ticks = reader.read_all(verify_crc=True)
115
+
116
+ # Filter by time range and contract token using block-level Bloom filters
117
+ subset = reader.query(
118
+ start_ns=int(ticks["timestamp_ns"][1000]),
119
+ end_ns=int(ticks["timestamp_ns"][5000]),
120
+ instrument_id=1001,
121
+ )
122
+ print(f"Retrieved {len(subset):,} filtered ticks in microseconds.")
123
+
124
+ # 4. Crash Recovery (in-place repair for unfinalized files)
125
+ report = vtf.recover_vtf_file("nifty_options.vtf")
126
+ print(f"Valid blocks preserved: {report.valid_blocks}, rows recovered: {report.valid_rows:,}")
127
+ ```
128
+
129
+ ---
130
+
131
+ ## 🛠️ The VTF Encoding Pipeline
132
+
133
+ ```
134
+ Raw Multiplexed Option Ticks (156 B/row, interleaved strikes)
135
+ │
136
+ ▼
137
+ Stage 1: Per-Instrument State Cache (512-slot hash table)
138
+ Delta-encoded against previous tick of SAME contract
139
+ │
140
+ ▼
141
+ Stage 2: Relational Order Book Ladder Encoding
142
+ Bid[0] vs LTP, Ask[0] spread, L2-L5 depth step diffs
143
+ │
144
+ ▼
145
+ Stage 3: ZigZag Variable-Length Delta Transformation
146
+ Maps small positive/negative jumps to small unsigned integers
147
+ │
148
+ ▼
149
+ Stage 4: SIMD SSSE3 4x4 Byte-Stream-Split (BSS) Transposition
150
+ Clusters upper 0x00 bytes into contiguous byte planes
151
+ │
152
+ ▼
153
+ Stage 5: High-Speed Block Codec (LZ4-Fast accel=4 / Zstd)
154
+ Instant hardware CRC32C verification on flush
155
+ ```
156
+
157
+ ---
158
+
159
+ ## 📦 Repository Structure
160
+
161
+ ```
162
+ VTF/
163
+ ├── vtf.py # Python API & NumPy ctypes wrapper
164
+ ├── vtf.hpp # C/C++ public API header
165
+ ├── vtf_core.cpp # Cross-platform zero-dependency C++17 engine
166
+ ├── benchmark_vtf.py # 500k tick benchmark suite against Parquet & Arrow
167
+ ├── example_quickstart.py # Ready-to-run quickstart example
168
+ ├── VTF_Specification_and_Benchmarks.md # Complete wire format specification
169
+ ├── setup.py # C++ extension setuptools builder
170
+ ├── pyproject.toml # PEP 517 build & cibuildwheel config
171
+ ├── LICENSE # MIT License
172
+ └── .github/workflows/
173
+ ├── ci.yml # Multi-OS CI testing (Linux, Windows, macOS)
174
+ └── release.yml # Automated multi-arch wheel build & PyPI publish
175
+ ```
176
+
177
+ ---
178
+
179
+ ## 🔬 Benchmark Verification
180
+
181
+ To run the complete benchmark suite comparing VTF with Parquet and Arrow:
182
+
183
+ ```bash
184
+ pip install numpy pyarrow
185
+ python benchmark_vtf.py
186
+ ```
187
+
188
+ ---
189
+
190
+ ## 📄 License
191
+
192
+ VTF is released under the [MIT License](LICENSE). Copyright (c) 2026 venkey2k.
nuro_vtf-1.0.6/nuro_vtf.egg-info/SOURCES.txt ADDED
@@ -0,0 +1,12 @@
1
+ LICENSE
2
+ README.md
3
+ nuro_vtf.py
4
+ pyproject.toml
5
+ setup.py
6
+ vtf.py
7
+ vtf_core.cpp
8
+ nuro_vtf.egg-info/PKG-INFO
9
+ nuro_vtf.egg-info/SOURCES.txt
10
+ nuro_vtf.egg-info/dependency_links.txt
11
+ nuro_vtf.egg-info/requires.txt
12
+ nuro_vtf.egg-info/top_level.txt
nuro_vtf-1.0.6/nuro_vtf.egg-info/requires.txt ADDED
@@ -0,0 +1 @@
1
+ numpy>=1.20
nuro_vtf-1.0.6/nuro_vtf.egg-info/top_level.txt ADDED
@@ -0,0 +1,3 @@
1
+ _libvtf
2
+ nuro_vtf
3
+ vtf
nuro_vtf-1.0.6/nuro_vtf.py ADDED
@@ -0,0 +1,9 @@
1
+ """
2
+ nuro_vtf — Convenience alias module for VTF (Venkey Tick Format).
3
+ Allows users to write either `import vtf` or `import nuro_vtf`.
4
+ """
5
+
6
+ from vtf import *
7
+ import vtf as _vtf
8
+
9
+ __all__ = [name for name in dir(_vtf) if not name.startswith("_")]
nuro_vtf-1.0.6/pyproject.toml ADDED
@@ -0,0 +1,61 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "nuro-vtf"
7
+ version = "1.0.6"
8
+
9
+ description = "VTF (Venkey Tick Format) v1.0 — High-Frequency Option Market Data Format & Zero-Copy Engine"
10
+ readme = "README.md"
11
+ license = "MIT"
12
+ authors = [
13
+ { name = "venkey2k", email = "venkey2kkid@gmail.com" }
14
+ ]
15
+ requires-python = ">=3.9"
16
+ dependencies = [
17
+ "numpy>=1.20"
18
+ ]
19
+ keywords = [
20
+ "hft",
21
+ "market-data",
22
+ "options",
23
+ "nifty",
24
+ "sensex",
25
+ "orderbook",
26
+ "greeks",
27
+ "binary-format",
28
+ "lz4",
29
+ "simd",
30
+ "zero-copy",
31
+ "vtf"
32
+ ]
33
+ classifiers = [
34
+ "Development Status :: 5 - Production/Stable",
35
+ "Intended Audience :: Financial and Insurance Industry",
36
+ "Topic :: Office/Business :: Financial :: Investment",
37
+ "Programming Language :: Python :: 3",
38
+ "Programming Language :: Python :: 3.9",
39
+ "Programming Language :: Python :: 3.10",
40
+ "Programming Language :: Python :: 3.11",
41
+ "Programming Language :: Python :: 3.12",
42
+ "Programming Language :: Python :: 3.13",
43
+ "Programming Language :: C++",
44
+ "Operating System :: POSIX :: Linux",
45
+ "Operating System :: Microsoft :: Windows",
46
+ "Operating System :: MacOS",
47
+ ]
48
+
49
+ [project.urls]
50
+ Homepage = "https://github.com/venkey2k/VTF"
51
+ Repository = "https://github.com/venkey2k/VTF.git"
52
+ Issues = "https://github.com/venkey2k/VTF/issues"
53
+
54
+ [tool.cibuildwheel]
55
+ build = "cp39-* cp310-* cp311-* cp312-* cp313-*"
56
+ skip = "*-musllinux_* pp* *_i686 *_win32"
57
+ manylinux-x86_64-image = "manylinux_2_28"
58
+ test-requires = "numpy"
59
+ test-command = "python -c \"import vtf; print('VTF itemsize:', vtf.VTF_TICK_DTYPE.itemsize)\""
60
+
61
+
nuro_vtf-1.0.6/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+