market-data-normalizer 1.3.1__py3-none-any.whl

market_data_normalizer-1.3.1.dist-info/METADATA

@@ -0,0 +1,330 @@
+Metadata-Version: 2.4
+Name: market-data-normalizer
+Version: 1.3.1
+Summary: Normalize heterogeneous market-data feeds (CSV, WebSocket JSON, FIX) into one exchange-agnostic schema.
+Project-URL: Homepage, https://harvestgroup360.com
+Project-URL: Repository, https://github.com/Harvestgroup360/market-data-normalizer
+Project-URL: Changelog, https://github.com/Harvestgroup360/market-data-normalizer/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/Harvestgroup360/market-data-normalizer/issues
+Author-email: HarvestGroup360 <github@harvestgroup360.com>
+License: MIT License
+        
+        Copyright (c) 2026 HarvestGroup360 (AMII LTD)
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: fix-protocol,market-data,normalization,quantitative-finance,tick-data,trading
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Office/Business :: Financial :: Investment
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# market-data-normalizer (`mdnorm`)
+
+[![CI](https://github.com/Harvestgroup360/market-data-normalizer/actions/workflows/ci.yml/badge.svg)](https://github.com/Harvestgroup360/market-data-normalizer/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+[![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](pyproject.toml)
+[![PyPI](https://img.shields.io/pypi/v/market-data-normalizer.svg)](https://pypi.org/project/market-data-normalizer/)
+
+Normalize heterogeneous market-data feeds — CSV tick dumps, exchange
+WebSocket JSON, and FIX — into a single, exchange-agnostic event schema, so
+downstream research and execution code never has to care where a tick came
+from.
+
+Zero runtime dependencies. Pure Python (3.10+). `Decimal` prices, integer
+nanosecond timestamps.
+
+## Why
+
+Every venue spells the same thing differently: `BTCUSDT` vs `XBT/USD`,
+millisecond epochs vs FIX `UTCTimestamp`, `is_buyer_maker` booleans vs side
+codes. Research notebooks and backtesters end up littered with per-venue
+parsing branches. `mdnorm` pushes that mess to the edge and hands the rest of
+your stack one clean type.
+
+## Install
+
+```console
+pip install market-data-normalizer
+```
+
+The distribution is named `market-data-normalizer`; the import name is
+`mdnorm`:
+
+```python
+import mdnorm
+```
+
+Pure Python, no runtime dependencies, Python 3.10+.
+
+## Quick start
+
+```python
+from mdnorm import from_csv_row, from_ws_json, from_fix
+
+# CSV row (ISO-8601 timestamp)
+from_csv_row(
+    {"symbol": "btc/usd", "ts": "2026-01-02T00:00:00Z",
+     "price": "42000.5", "size": "0.25", "side": "buy"},
+    venue="coinbase",
+)
+
+# Exchange WebSocket trade message
+from_ws_json({"s": "BTCUSDT", "p": "42000.5", "q": "0.25",
+              "T": 1767312000000, "m": False}, venue="binance")
+
+# FIX execution report (SOH-delimited in the wild; "|" here for readability)
+from_fix("55=BTC/USD|31=42000.5|32=0.25|54=1|60=20260102-00:00:00",
+         venue="lmax", sep="|")
+```
+
+All three calls above produce the **same** `MarketEvent`.
+
+### Quotes (bid/ask)
+
+```python
+from mdnorm import from_ws_quote
+
+q = from_ws_quote(
+    {"s": "BTCUSDT", "b": "41999.5", "B": "1.2",
+     "a": "42000.5", "A": "0.8", "T": 1767312000000},
+    venue="binance",
+)
+q.mid_price   # Decimal("42000.0")
+q.spread      # Decimal("1.0")
+```
+
+`from_csv_quote` does the same for CSV rows with bid/ask columns.
+
+### OHLCV bars
+
+```python
+from mdnorm import time_bars
+
+bars = time_bars(events, interval_ns=60_000_000_000)  # 1-minute bars
+bars[0].open, bars[0].high, bars[0].low, bars[0].close, bars[0].volume, bars[0].vwap
+```
+
+`time_bars` reduces a stream of trade events into fixed-interval OHLCV `Bar`s
+(with VWAP and trade count), sorting out-of-order input and skipping quotes.
+
+`resample_bars(bars, interval_ns)` downsamples bars to a coarser interval
+(e.g. 1-minute → 5-minute) with correct OHLC aggregation and volume-weighted
+VWAP.
+
+`fill_gaps(bars)` returns a gapless series, inserting flat zero-volume bars
+(OHLC = previous close) for any interval with no trades — a continuous grid for
+backtests and feature pipelines.
+
+### Event-driven bars
+
+Time bars are not the only clock. Sample by activity instead:
+
+```python
+from decimal import Decimal
+from mdnorm import count_bars, volume_bars, dollar_bars
+
+count_bars(events, every=500)                       # tick bars
+volume_bars(events, min_volume=Decimal("100"))      # volume bars
+dollar_bars(events, min_notional=Decimal("1e6"))    # dollar bars
+```
+
+### Trading sessions
+
+Filter a feed down to the hours that matter, with daylight saving handled
+for you:
+
+```python
+from mdnorm import US_EQUITY_RTH, filter_session, group_by_session_date
+
+rth = filter_session(events, US_EQUITY_RTH)        # 09:30-16:00 New York
+by_day = group_by_session_date(events, US_EQUITY_RTH)
+```
+
+Overnight windows (a session that opens at 18:00 and closes at 17:00 the
+next day) are supported, and `session_date` keeps a whole night in one
+bucket. From the command line:
+
+```console
+$ mdnorm bars trades.csv --interval 5m --session 09:30-16:00 --tz America/New_York -o rth.csv
+```
+
+### Data quality
+
+```python
+from mdnorm.quality import find_issues, clean
+
+find_issues(events)          # list of QualityIssue (outlier / gap / out_of_order / non_positive)
+cleaned, issues = clean(events)  # drop bad ticks & invalid rows, keep a report
+```
+
+`clean` removes price outliers and non-positive price/size records and returns
+the surviving events plus everything it flagged.
+
+### Serialization
+
+```python
+from mdnorm import to_records
+
+to_records(events)                 # list of flat dicts (Decimals as strings)
+to_records(bars, as_float=True)    # numeric output for DataFrames
+```
+
+`to_records` (and `event_to_dict` / `bar_to_dict`) flatten events and bars into
+plain, JSON-serialisable dicts — drop straight into `pandas.DataFrame`, a
+`csv.DictWriter`, or `json.dumps`.
+
+### Consolidating streams
+
+```python
+from mdnorm import merge_streams, dedupe
+
+timeline = dedupe(merge_streams(binance_events, coinbase_events))
+```
+
+`merge_streams` interleaves multiple venue feeds into one timestamp-ordered
+timeline; `dedupe` drops exact duplicate events left behind by reconnects and
+replays.
+
+### CSV files
+
+```python
+from mdnorm import read_csv_trades, write_records_csv
+
+events = read_csv_trades("trades.csv", venue="coinbase")   # file -> events
+write_records_csv(bars, "bars.csv", as_float=True)          # events/bars -> file
+```
+
+`read_csv_trades` parses a whole CSV of trades into normalized events;
+`write_records_csv` writes events or bars back out. Standard library only.
+
+### NDJSON / JSON Lines
+
+```python
+from mdnorm import write_jsonl, read_jsonl_events
+
+write_jsonl(events, "events.jsonl")          # one JSON object per line
+events2 = read_jsonl_events("events.jsonl")  # lossless round-trip
+
+# large files: stream lazily, .gz handled transparently
+for e in iter_jsonl_events("dump.jsonl.gz"):
+    ...
+```
+
+### Pipelines
+
+Declare a processing chain once, reuse it everywhere:
+
+```python
+from decimal import Decimal
+from mdnorm import Pipeline
+
+pipe = (
+    Pipeline()
+    .dedupe()
+    .clean(max_return=Decimal("0.1"))
+    .time_bars(60_000_000_000)   # 1-minute bars
+    .fill_gaps()
+)
+bars = pipe.run(events)
+print(pipe.last_issues)          # quality report from clean()
+```
+
+### Command line
+
+The common conversions ship as a zero-dependency CLI:
+
+```console
+$ mdnorm bars trades.csv --venue binance --interval 1m -o bars.csv
+$ mdnorm quality trades.csv --max-gap 5m
+$ mdnorm convert trades.csv -o trades.jsonl
+```
+
+Also available as `python -m mdnorm`.
+
+## The unified schema
+
+```python
+@dataclass(frozen=True, slots=True)
+class MarketEvent:
+    symbol: str          # canonical "BASE-QUOTE", e.g. "BTC-USD"
+    venue: str           # source venue
+    event_type: EventType  # TRADE | QUOTE
+    ts_ns: int           # nanoseconds since Unix epoch (UTC)
+    price: Decimal | None
+    size:  Decimal | None
+    side:  Side | None     # BUY | SELL
+    # ... plus bid/ask fields for quotes
+```
+
+## Design notes
+
+- **Money is `Decimal`.** Prices and sizes never touch binary floats, so
+  `42000.10` stays `42000.10`.
+- **Time is integer nanoseconds, UTC.** One comparable integer regardless of
+  whether the source gave seconds, milliseconds, or a FIX timestamp string.
+- **Symbols are canonicalized** to `BASE-QUOTE`, with venue aliases resolved
+  (`XBT` → `BTC`) and quote currencies detected longest-match-first so
+  `USDT` wins over `USD`.
+- **Normalizers are pure functions** — one raw record in, one `MarketEvent`
+  out — which keeps them trivial to unit-test and compose into any streaming
+  or batch pipeline.
+
+## Architecture
+
+```
+raw feed ──► normalizer ─────────────► MarketEvent ──► your pipeline
+ (CSV /      (from_csv_row /            (unified,       (research,
+  WS JSON /   from_ws_json /             immutable)      backtest,
+  FIX)        from_fix)                                  execution)
+                    │
+                    ├── symbols.canonical_symbol()   BTCUSDT → BTC-USDT
+                    └── timeutil.*_to_ns()           any time → ns UTC
+```
+
+## Tests
+
+```bash
+pip install pytest
+pytest -q
+```
+
+The suite includes a cross-venue equivalence test proving CSV, WebSocket and
+FIX representations of one trade collapse to an identical event.
+
+## License
+
+MIT © HarvestGroup360 (AMII LTD). See [LICENSE](LICENSE).
+
+---
+
+Maintained by [HarvestGroup360](https://harvestgroup360.com) as part of our
+open quantitative-infrastructure tooling.

market_data_normalizer-1.3.1.dist-info/RECORD

@@ -0,0 +1,21 @@
+mdnorm/__init__.py,sha256=64_G8P30VnYexQt73U4WitHs8MA62yO3qtL8cQJh08s,2057
+mdnorm/__main__.py,sha256=gsrjzHcpdagrMBi_jZpO8KJDStEjeeTuUfbFn8Jo8Hk,135
+mdnorm/bars.py,sha256=Koqv7rKqdfQzwWFWZNxfOaXHV-gBoW2Rfz7ZpOp-pdw,8635
+mdnorm/cli.py,sha256=TXpFxLh4-UKH21Fgqmu1803q_Qc16A-YZLKCbQaI2NU,8518
+mdnorm/csvio.py,sha256=t66oKe7y-MokpA_iCCDdk1-wvPPNeysxBTnL1j82JGc,2461
+mdnorm/fileio.py,sha256=0nXi45qca3aHx5H_jtXPvApVASHMttdxt4Sbo1lDZt0,731
+mdnorm/jsonl.py,sha256=6kXYduWczuintgABDI1JJm2u2_K0qIUT0cRHRlf92YE,3410
+mdnorm/normalizers.py,sha256=3AKiyxAMTh-lT0hpQL266riS4Cjfmbiz3GAWtMW2geM,6106
+mdnorm/pipeline.py,sha256=E7bwB2CHKMBzXV-PP_gOAVeRAuR99135EOXnjh_ahrE,5128
+mdnorm/quality.py,sha256=2AbNZErGOvZWlGn-2AFV3hrkJRepcql7eLX2b6WCzP8,3176
+mdnorm/records.py,sha256=xj5jBkPEjqtIZaCD7iRwQaK9ZdoeZfFrSpYcFzBmfLI,2415
+mdnorm/schema.py,sha256=ujlvpPgd_-6-ePXszAzapREmVlIHJJZpuJEVw9j4VCI,2262
+mdnorm/sessions.py,sha256=I9voJ5WqHafFLYn70HoUSQtynI4ycsxsvf-QT2jO9ng,6927
+mdnorm/streams.py,sha256=e-oHhSK8Fj86a0BhNf7IitB9AeIvTyUU1LSUghWGrm0,1248
+mdnorm/symbols.py,sha256=j6mJMjW5i-4w6uDF8Aj3bl6XahH2H5x_ClIXmp9nJX8,1968
+mdnorm/timeutil.py,sha256=MVv61eHgTWZeiie4PzAw8uhHu2BrdYiYV9ti6R5A8EQ,1525
+market_data_normalizer-1.3.1.dist-info/METADATA,sha256=SCeVGy7UIGiFOIBGr0orRHKwElUDVZ-QHZAWhxbjTGk,11267
+market_data_normalizer-1.3.1.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+market_data_normalizer-1.3.1.dist-info/entry_points.txt,sha256=n1EgYTITUpYmGCfAtrmrFVuGS0Frf691iSQlurpTUQY,43
+market_data_normalizer-1.3.1.dist-info/licenses/LICENSE,sha256=HJNlUNuaSm-lzx02gw6NJPzrpidtG58PYZfLdTv1w7Y,1083
+market_data_normalizer-1.3.1.dist-info/RECORD,,

market_data_normalizer-1.3.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

market_data_normalizer-1.3.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+mdnorm = mdnorm.cli:main

market_data_normalizer-1.3.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 HarvestGroup360 (AMII LTD)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mdnorm/__init__.py

@@ -0,0 +1,92 @@
+"""market-data-normalizer (mdnorm).
+
+Normalize heterogeneous market-data feeds (CSV, exchange WebSocket JSON, FIX)
+into a single, exchange-agnostic :class:`MarketEvent` schema.
+"""
+from __future__ import annotations
+
+from .bars import (
+    Bar,
+    count_bars,
+    dollar_bars,
+    fill_gaps,
+    resample_bars,
+    time_bars,
+    volume_bars,
+)
+from .csvio import iter_csv_trades, read_csv_trades, write_records_csv
+from .jsonl import event_from_dict, iter_jsonl_events, read_jsonl_events, write_jsonl
+from .normalizers import (
+    from_csv_quote,
+    from_csv_row,
+    from_fix,
+    from_ws_json,
+    from_ws_quote,
+)
+from .pipeline import Pipeline
+from .quality import QualityIssue, clean, find_issues
+from .records import bar_to_dict, event_to_dict, to_records
+from .schema import EventType, MarketEvent, Side
+from .sessions import (
+    US_EQUITY_RTH,
+    US_FUTURES_OVERNIGHT,
+    WEEKDAYS,
+    Session,
+    filter_session,
+    group_by_session_date,
+    in_session,
+    parse_session,
+    session_date,
+)
+from .streams import dedupe, merge_streams
+from .symbols import canonical_symbol
+from .timeutil import epoch_to_ns, fix_utc_to_ns, iso_to_ns
+
+__version__ = "1.3.1"
+
+__all__ = [
+    "MarketEvent",
+    "EventType",
+    "Side",
+    "from_csv_row",
+    "from_ws_json",
+    "from_fix",
+    "from_ws_quote",
+    "from_csv_quote",
+    "Bar",
+    "time_bars",
+    "resample_bars",
+    "fill_gaps",
+    "count_bars",
+    "volume_bars",
+    "dollar_bars",
+    "Session",
+    "in_session",
+    "filter_session",
+    "session_date",
+    "group_by_session_date",
+    "parse_session",
+    "WEEKDAYS",
+    "US_EQUITY_RTH",
+    "US_FUTURES_OVERNIGHT",
+    "QualityIssue",
+    "find_issues",
+    "clean",
+    "event_to_dict",
+    "bar_to_dict",
+    "to_records",
+    "merge_streams",
+    "dedupe",
+    "read_csv_trades",
+    "iter_csv_trades",
+    "write_records_csv",
+    "read_jsonl_events",
+    "iter_jsonl_events",
+    "write_jsonl",
+    "event_from_dict",
+    "Pipeline",
+    "canonical_symbol",
+    "epoch_to_ns",
+    "iso_to_ns",
+    "fix_utc_to_ns",
+]

mdnorm/__main__.py

@@ -0,0 +1,5 @@
+"""Allow ``python -m mdnorm ...`` to invoke the CLI."""
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())