opencode-pyneruntime 6.6.4__py3-none-any.whl

pynecore/core/ohlcv_file.py

@@ -0,0 +1,1888 @@
+"""
+Fast and efficient OHLCV data reader/writer
+
+The file is a binary file with the following 24 bytes structure:
+ - timestamp: uint32 (4 bytes) - good until 2106 (I will fix this then, I promise ;))
+ - open:     float32 (4 bytes)
+ - high:     float32 (4 bytes)
+ - low:      float32 (4 bytes)
+ - close:    float32 (4 bytes)
+ - volume:   float32 (4 bytes)
+
+The .ohlcv format cannot have gaps in it. All gaps are filled with the previous close price and -1 volume.
+"""
+from typing import Iterator
+
+import csv
+import json
+import math
+import mmap
+import os
+import re
+import struct
+from collections import Counter
+from datetime import datetime, time, timedelta, timezone as dt_timezone, UTC
+from io import BufferedWriter, BufferedRandom
+from math import gcd as math_gcd
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+from pynecore.types.ohlcv import OHLCV
+from ..core.syminfo import SymInfoInterval
+
+RECORD_SIZE = 24  # 6 * 4
+STRUCT_FORMAT = 'Ifffff'  # I: uint32, f: float32
+
+_QTY_STEP_MIN_SAMPLES = 100  # positive-volume bars needed before analyzed_qty_step answers
+_QTY_STEP_MAX_DECIMALS = 8  # more decimals than any real exchange lot step -> float dust
+
+__all__ = ['OHLCVWriter', 'OHLCVReader', 'RECORD_SIZE', 'STRUCT_FORMAT']
+
+
+def _format_float(value: float) -> str:
+    """Format float with max 8 decimal places, removing trailing zeros"""
+    return f"{value:.8g}"
+
+
+def _parse_timezone_param(tz: str | None) -> dt_timezone | ZoneInfo | None:
+    """
+    Parse timezone parameter into a timezone object.
+
+    :param tz: Timezone string (e.g. 'UTC', 'Europe/London', '+0100', '-0500')
+    :return: Timezone object or None if tz is None
+    :raises ValueError: If timezone format is invalid
+    """
+    if not tz:
+        return None
+
+    if tz.startswith(('+', '-')):
+        # Handle UTC offset format (e.g. +0100, -0500)
+        sign = 1 if tz.startswith('+') else -1
+        hours = int(tz[1:3])
+        minutes = int(tz[3:]) if len(tz) > 3 else 0
+        return dt_timezone(sign * timedelta(hours=hours, minutes=minutes))
+    else:
+        # Handle named timezone (e.g. UTC, Europe/London)
+        try:
+            return ZoneInfo(tz)
+        except Exception as e:
+            raise ValueError(f"Invalid timezone {tz}: {e}")
+
+
+def _find_timestamp_columns(headers: list[str],
+                            timestamp_column: str | None = None,
+                            date_column: str | None = None,
+                            time_column: str | None = None) -> tuple[int | None, int | None, int | None]:
+    """
+    Find timestamp-related column indices in headers.
+
+    :param headers: List of column headers (already lowercased)
+    :param timestamp_column: Optional specific timestamp column name
+    :param date_column: Optional date column name (when split into date+time)
+    :param time_column: Optional time column name (when split into date+time)
+    :return: Tuple of (timestamp_idx, date_idx, time_idx)
+    :raises ValueError: If required columns are not found
+    """
+    timestamp_idx = None
+    date_idx = None
+    time_idx = None
+
+    if date_column and time_column:
+        try:
+            date_idx = headers.index(date_column.lower())
+            time_idx = headers.index(time_column.lower())
+        except ValueError:
+            raise ValueError(f"Date/time columns not found: {date_column}/{time_column}")
+    else:
+        timestamp_col = timestamp_column.lower() if timestamp_column else None
+        if timestamp_col:
+            try:
+                timestamp_idx = headers.index(timestamp_col)
+            except ValueError:
+                raise ValueError(f"Timestamp column not found: {timestamp_col}")
+        else:
+            # Try common names (ts_event / ts_recv are Databento's nanosecond timestamps)
+            for col in ['timestamp', 'time', 'date', 'ts_event', 'ts_recv']:
+                try:
+                    timestamp_idx = headers.index(col)
+                    break
+                except ValueError:
+                    continue
+
+            if timestamp_idx is None:
+                raise ValueError("Timestamp column not found!")
+
+    return timestamp_idx, date_idx, time_idx
+
+
+def _find_ohlcv_columns(headers: list[str]) -> tuple[int, int, int, int, int]:
+    """
+    Find OHLCV column indices in headers.
+
+    :param headers: List of column headers (already lowercased)
+    :return: Tuple of (open_idx, high_idx, low_idx, close_idx, volume_idx)
+    :raises ValueError: If required columns are not found
+    """
+    try:
+        o_idx = headers.index('open')
+        h_idx = headers.index('high')
+        l_idx = headers.index('low')
+        c_idx = headers.index('close')
+        v_idx = headers.index('volume')
+    except ValueError as e:
+        raise ValueError(f"Missing required column: {str(e)}")
+
+    return o_idx, h_idx, l_idx, c_idx, v_idx
+
+
+def _parse_timestamp(ts_str: str, timestamp_format: str | None = None, timezone=None) -> int:
+    """
+    Parse timestamp string to Unix timestamp.
+
+    :param ts_str: Timestamp string to parse
+    :param timestamp_format: Optional specific datetime format for parsing
+    :param timezone: Optional timezone to apply to the parsed datetime
+    :return: Unix timestamp as integer
+    :raises ValueError: If timestamp cannot be parsed
+    """
+    # Handle numeric timestamps (seconds / ms / us / ns since epoch)
+    if ts_str.isdigit():
+        timestamp = int(ts_str)
+        # Repeatedly downscale by 1000 until we land in the valid seconds range.
+        # Covers ms (13 digits), us (16 digits) and ns (19 digits, e.g. Databento).
+        while timestamp > 253402300799:  # 9999-12-31 23:59:59
+            timestamp //= 1000
+        return timestamp
+
+    # Truncate sub-microsecond fractional digits (Databento emits nanoseconds,
+    # e.g. "2026-02-23T00:00:00.000000000Z" — Python's %f only matches 1-6 digits).
+    ts_str = re.sub(r'(\.\d{6})\d+', r'\1', ts_str)
+
+    # Parse datetime string
+    dt = None
+    if timestamp_format:
+        dt = datetime.strptime(ts_str, timestamp_format)
+    else:
+        # Try common formats
+        for fmt in [
+            '%Y-%m-%d %H:%M:%S%z',  # 2024-01-08 19:00:00+0000
+            '%Y-%m-%d %H:%M:%S%Z',  # 2024-01-08 19:00:00UTC
+            '%Y-%m-%dT%H:%M:%S%z',  # 2024-01-08T19:00:00+0000
+            '%Y-%m-%dT%H:%M:%S.%f%z',  # 2024-01-08T19:00:00.123456+0000
+            '%Y-%m-%dT%H:%M:%S.%fZ',  # 2026-02-23T00:00:00.000000Z (Databento)
+            '%Y-%m-%d %H:%M:%S',
+            '%Y/%m/%d %H:%M:%S',
+            '%d.%m.%Y %H:%M:%S',
+            '%Y-%m-%dT%H:%M:%S',
+            '%Y-%m-%dT%H:%M:%SZ',  # ISO with Z
+            '%Y-%m-%d %H:%M',
+            '%Y%m%d %H:%M:%S'
+        ]:
+            try:
+                dt = datetime.strptime(ts_str, fmt)
+                break
+            except ValueError:
+                continue
+
+        if dt is None:
+            raise ValueError(f"Could not parse timestamp: {ts_str}")
+
+    # Apply timezone parameter only if the datetime doesn't already have timezone info
+    # This preserves timezone information from the timestamp string itself
+    if timezone and dt is not None and dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone)
+
+    assert dt is not None
+    return int(dt.timestamp())
+
+
+class OHLCVWriter:
+    """
+    Binary OHLCV data writer using direct file operations
+    """
+
+    __slots__ = ('path', '_file', '_size', '_start_timestamp', '_interval', '_current_pos', '_last_timestamp',
+                 '_price_changes', '_price_decimals', '_last_close', '_analyzed_tick_size',
+                 '_analyzed_price_scale', '_analyzed_min_move', '_confidence',
+                 '_volume_max_decimals', '_volume_count', '_volume_dust_count',
+                 '_trading_hours', '_analyzed_opening_hours', '_timestamp_offsets', '_analyzed_timezone',
+                 '_truncate',
+                 '_extra_file', '_extra_writer', '_extra_headers', '_extra_row_count')
+
+    def __init__(self, path: str | Path, truncate: bool = False):
+        self.path: str = str(path)
+        self._file: BufferedWriter | BufferedRandom | None = None
+        self._truncate: bool = truncate
+        self._size: int = 0
+        self._start_timestamp: int | None = None
+        self._interval: int | None = None
+        self._current_pos: int = 0
+        self._last_timestamp: int | None = None
+        # Tick size analysis
+        self._price_changes: list[float] = []
+        self._price_decimals: set[int] = set()
+        self._last_close: float | None = None
+        self._analyzed_tick_size: float | None = None
+        self._analyzed_price_scale: int | None = None
+        self._analyzed_min_move: int | None = None
+        self._confidence: float = 0.0
+        # Quantity step (mincontract) analysis
+        self._volume_max_decimals: int = 0
+        self._volume_count: int = 0
+        self._volume_dust_count: int = 0
+        # Trading hours analysis
+        self._trading_hours: dict[tuple[int, int], int] = {}  # (weekday, hour) -> count
+        self._analyzed_opening_hours: list | None = None
+        # Extra fields sidecar CSV
+        self._extra_file = None
+        self._extra_writer = None
+        self._extra_headers: list[str] | None = None
+        self._extra_row_count: int = 0
+
+    def __enter__(self):
+        self.open()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    @property
+    def is_open(self) -> bool:
+        """
+        Check if file is open
+        """
+        return self._file is not None
+
+    @property
+    def size(self) -> int:
+        """
+        Number of records in the file
+        """
+        return self._size
+
+    @property
+    def start_timestamp(self) -> int | None:
+        """
+        Timestamp of the first record
+        """
+        return self._start_timestamp
+
+    @property
+    def start_datetime(self) -> datetime:
+        """
+        Datetime of the first record
+        """
+        assert self._start_timestamp is not None
+        return datetime.fromtimestamp(self._start_timestamp, UTC)
+
+    @property
+    def end_timestamp(self) -> int | None:
+        """
+        Timestamp of the last record
+        """
+        if self._start_timestamp is None or self._interval is None:
+            return None
+        return self._start_timestamp + self._interval * (self._size - 1)
+
+    @property
+    def end_datetime(self) -> datetime | None:
+        """
+        Datetime of the last record
+        """
+        ts = self.end_timestamp
+        if ts is None:
+            return None
+        return datetime.fromtimestamp(ts, UTC)
+
+    @property
+    def interval(self) -> int | None:
+        """
+        Interval between records
+        """
+        return self._interval
+
+    @property
+    def analyzed_tick_size(self) -> float | None:
+        """
+        Automatically detected tick size from price data
+        """
+        if self._analyzed_tick_size is None and len(self._price_changes) >= 10:
+            self._analyze_tick_size()
+        return self._analyzed_tick_size
+
+    @property
+    def analyzed_price_scale(self) -> int | None:
+        """
+        Automatically detected price scale from price data
+        """
+        if self._analyzed_price_scale is None and len(self._price_changes) >= 10:
+            self._analyze_tick_size()
+        return self._analyzed_price_scale
+
+    @property
+    def analyzed_min_move(self) -> int | None:
+        """
+        Automatically detected min move (usually 1)
+        """
+        if self._analyzed_min_move is None and len(self._price_changes) >= 10:
+            self._analyze_tick_size()
+        return self._analyzed_min_move
+
+    @property
+    def tick_analysis_confidence(self) -> float:
+        """
+        Confidence of tick size analysis (0.0 to 1.0)
+        """
+        if self._confidence == 0.0 and len(self._price_changes) >= 10:
+            self._analyze_tick_size()
+        return self._confidence
+
+    @property
+    def analyzed_opening_hours(self) -> list | None:
+        """
+        Automatically detected opening hours from trading activity
+        Returns list of SymInfoInterval tuples or None if not enough data
+        """
+        if self._analyzed_opening_hours is None and self._has_enough_data_for_opening_hours():
+            self._analyze_opening_hours()
+        return self._analyzed_opening_hours
+
+    @property
+    def analyzed_qty_step(self) -> float | None:
+        """
+        Automatically detected quantity step (``mincontract`` candidate) from volume data.
+
+        Every bar volume is a sum of trade quantities on the exchange's lot
+        grid, so the maximum decimal precision seen across the written volumes
+        bounds the lot step from below. Returns ``None`` without enough
+        positive-volume bars or when the volumes carry float dust (which means
+        they were computed, not exchange-native).
+        """
+        if self._volume_count < _QTY_STEP_MIN_SAMPLES:
+            return None
+        if self._volume_dust_count * 20 > self._volume_count:  # >5% dusty samples
+            return None
+        return 10.0 ** -self._volume_max_decimals
+
+    def open(self) -> 'OHLCVWriter':
+        """
+        Open file for writing
+        """
+        # If truncate is True, always open in write mode to clear existing data
+        if self._truncate:
+            self._file = open(self.path, 'wb+')
+        else:
+            # Open in rb+ mode to allow both reading and writing
+            self._file = open(self.path, 'rb+') if os.path.exists(self.path) else open(self.path, 'wb+')
+        self._size = os.path.getsize(self.path) // RECORD_SIZE
+
+        # Read initial metadata if file exists
+        assert self._file is not None
+        if self._size >= 2:
+            self._file.seek(0)
+            data = self._file.read(4)
+            first_timestamp = struct.unpack('I', data)[0]
+            self._file.seek(RECORD_SIZE)
+            data = self._file.read(4)
+            second_timestamp = struct.unpack('I', data)[0]
+            self._start_timestamp = first_timestamp
+            self._interval = second_timestamp - first_timestamp
+            assert self._interval is not None
+            self._last_timestamp = first_timestamp + self._interval * (self._size - 1)
+
+        # Position at end for appending
+        self._file.seek(0, os.SEEK_END)
+        self._current_pos = self._size
+
+        # Collect trading hours from existing data for analysis
+        if self._size > 0 and not self._truncate:
+            self._collect_existing_trading_hours()
+
+        # Check for existing extra fields sidecar
+        extra_path = Path(self.path).with_suffix('.extra.csv')
+        if extra_path.exists():
+            with open(extra_path, 'r', newline='') as ef:
+                reader = csv.reader(ef)
+                headers = next(reader, None)
+                if headers:
+                    self._extra_headers = headers
+                    self._extra_row_count = sum(1 for _ in reader)
+
+        return self
+
+    def write(self, candle: OHLCV) -> None:
+        """
+        Write a single OHLCV candle at current position.
+        If there is a gap between current and previous timestamp,
+        fills it with the previous close price and -1 volume to indicate gap filling.
+
+        :param candle: OHLCV data to write
+        """
+        if self._file is None:
+            raise IOError("File not opened!")
+
+        if self._size == 0:
+            self._start_timestamp = candle.timestamp
+        elif self._size == 1:
+            # First interval detection
+            assert self._start_timestamp is not None
+            interval = candle.timestamp - self._start_timestamp
+            self._interval = interval
+            if interval == 0:
+                raise ValueError(
+                    f"Duplicate timestamp {candle.timestamp} on consecutive rows. "
+                    f"Input data must contain a single bar per timestamp "
+                    f"(e.g. for Databento, pre-filter to one publisher_id / instrument_id).")
+            if interval < 0:
+                raise ValueError(
+                    f"Timestamps must be in chronological order. "
+                    f"Got {candle.timestamp} after {self._start_timestamp}.")
+        elif self._size >= 2:  # Changed from elif self._size == 2: to properly handle all cases
+            # Check chronological order
+            if self._last_timestamp is not None and candle.timestamp == self._last_timestamp:
+                raise ValueError(
+                    f"Duplicate timestamp {candle.timestamp} on consecutive rows. "
+                    f"Input data must contain a single bar per timestamp "
+                    f"(e.g. for Databento, pre-filter to one publisher_id / instrument_id).")
+            if self._last_timestamp is not None and candle.timestamp < self._last_timestamp:
+                raise ValueError(
+                    f"Timestamps must be in chronological order. "
+                    f"Got {candle.timestamp} after {self._last_timestamp}.")
+
+            # Check if we found a smaller interval (indicates initial interval was wrong due to gap)
+            if self._interval is not None and self._last_timestamp is not None:
+                current_interval = candle.timestamp - self._last_timestamp
+
+                # If we find a smaller interval, the initial one was wrong (had a gap)
+                if 0 < current_interval < self._interval:
+                    # Rebuild file with correct interval
+                    self._rebuild_with_correct_interval(current_interval)
+                    # Now write the current candle with the corrected setup
+                    self.write(candle)
+                    return
+
+            # Calculate expected timestamp and fill gaps
+            if self._interval is not None and self._last_timestamp is not None:
+                expected_ts = self._last_timestamp + self._interval
+
+                # Fill gap if needed
+                if candle.timestamp > expected_ts:
+                    # Get previous candle's close price
+                    self._file.seek((self._current_pos - 1) * RECORD_SIZE)
+                    prev_record = self._file.read(RECORD_SIZE)
+                    prev_data = struct.unpack(STRUCT_FORMAT, prev_record)
+                    prev_close = prev_data[4]  # 4th index is close price
+
+                    # Fill gap with previous close and -1 volume (gap indicator)
+                    while expected_ts < candle.timestamp:
+                        gap_data = struct.pack(STRUCT_FORMAT,
+                                                       expected_ts, prev_close, prev_close,
+                                                       prev_close, prev_close, -1.0)
+                        self._file.seek(self._current_pos * RECORD_SIZE)
+                        self._file.write(gap_data)
+                        self._write_extra_gap()
+                        self._current_pos += 1
+                        self._size = max(self._size, self._current_pos)
+                        expected_ts += self._interval
+
+        # Write actual data
+        self._file.seek(self._current_pos * RECORD_SIZE)
+        data = struct.pack(STRUCT_FORMAT,
+                           candle.timestamp, candle.open, candle.high,
+                           candle.low, candle.close, candle.volume)
+        self._file.write(data)
+        self._file.flush()
+
+        # Write extra fields to sidecar CSV
+        self._write_extra_data(candle.extra_fields)
+
+        # Collect data for tick size analysis
+        self._collect_price_data(candle)
+
+        # Collect volume data for quantity step analysis
+        self._collect_volume_data(candle)
+
+        # Collect trading hours data
+        self._collect_trading_hours(candle)
+
+        self._last_timestamp = candle.timestamp
+        self._current_pos += 1
+        self._size = max(self._size, self._current_pos)
+
+    def seek_to_timestamp(self, timestamp: int) -> None:
+        """
+        Move write position to specific timestamp.
+        Uses interval between bars to calculate position.
+        """
+        if self._interval is None or self._start_timestamp is None:
+            return
+
+        if timestamp < self._start_timestamp:
+            raise ValueError("Timestamp before start of data")
+
+        record_num = (timestamp - self._start_timestamp) // self._interval
+        self.seek(int(record_num))
+
+    def seek(self, position: int) -> None:
+        """
+        Move write position to specific record number
+        """
+        if position < 0:
+            raise ValueError("Negative position not allowed")
+        assert self._file is not None
+
+        self._current_pos = position
+        self._file.seek(position * RECORD_SIZE)
+
+    def truncate(self) -> None:
+        """
+        Truncate file at current position.
+        All data after current position will be deleted.
+        """
+        if self._file is None:
+            raise IOError("File not opened!")
+
+        # Calculate new size in bytes
+        new_size = self._current_pos * RECORD_SIZE
+
+        # Truncate the file
+        self._file.truncate(new_size)
+        self._size = self._current_pos
+
+        # Update interval if we deleted too much
+        if self._size < 2:
+            self._interval = None
+            if self._size == 0:
+                self._start_timestamp = None
+
+        # Clean up extra fields sidecar on full truncate
+        if self._current_pos == 0:
+            self._close_extra_csv()
+            extra_path = Path(self.path).with_suffix('.extra.csv')
+            if extra_path.exists():
+                extra_path.unlink()
+            self._extra_headers = None
+            self._extra_row_count = 0
+
+    def close(self):
+        """
+        Close the file
+        """
+        if self._file:
+            self._file.close()
+            self._file = None
+        self._close_extra_csv()
+
+    def _close_extra_csv(self) -> None:
+        """Close the extra fields sidecar CSV if open."""
+        if self._extra_file:
+            self._extra_file.close()
+            self._extra_file = None
+            self._extra_writer = None
+
+    def _open_extra_csv(self, headers: list[str]) -> None:
+        """Open the extra fields sidecar CSV for writing."""
+        extra_path = Path(self.path).with_suffix('.extra.csv')
+
+        if (self._extra_headers == headers
+                and self._extra_row_count <= self._current_pos):
+            # Compatible header exists, append and pad if needed
+            self._extra_file = open(extra_path, 'a', newline='')
+            self._extra_writer = csv.writer(self._extra_file)
+            empty = [''] * len(headers)
+            for _ in range(self._current_pos - self._extra_row_count):
+                self._extra_writer.writerow(empty)
+            self._extra_row_count = self._current_pos
+        else:
+            # New file or incompatible header
+            self._extra_file = open(extra_path, 'w', newline='')
+            self._extra_writer = csv.writer(self._extra_file)
+            self._extra_headers = headers
+            self._extra_writer.writerow(headers)
+            empty = [''] * len(headers)
+            for _ in range(self._current_pos):
+                self._extra_writer.writerow(empty)
+            self._extra_row_count = self._current_pos
+
+    def _write_extra_gap(self) -> None:
+        """Write an empty row to the extra CSV for a gap-fill position."""
+        if self._extra_writer is not None:
+            self._extra_writer.writerow([''] * len(self._extra_headers))
+            self._extra_row_count += 1
+
+    def _write_extra_data(self, extra_fields: dict | None) -> None:
+        """Write extra fields data row to the sidecar CSV."""
+        if extra_fields and self._extra_writer is None:
+            self._open_extra_csv(list(extra_fields.keys()))
+        if self._extra_writer is not None:
+            if extra_fields:
+                row = []
+                for h in self._extra_headers:
+                    v = extra_fields.get(h)
+                    if v is None:
+                        row.append('')
+                    elif isinstance(v, float):
+                        row.append(_format_float(v))
+                    else:
+                        row.append(str(v))
+                self._extra_writer.writerow(row)
+            else:
+                self._extra_writer.writerow([''] * len(self._extra_headers))
+            self._extra_row_count += 1
+            self._extra_file.flush()
+
+    def _collect_price_data(self, candle: OHLCV) -> None:
+        """
+        Collect price data for tick size analysis during writing.
+        """
+        # Collect price changes
+        if self._last_close is not None:
+            change = abs(candle.close - self._last_close)
+            if change > 0 and len(self._price_changes) < 1000:  # Limit to 1000 samples
+                self._price_changes.append(change)
+
+        # Collect decimal places
+        for price in [candle.open, candle.high, candle.low, candle.close]:
+            if price != int(price):  # Has decimal component
+                price_str = f"{price:.15f}".rstrip('0').rstrip('.')
+                if '.' in price_str:
+                    decimals = len(price_str.split('.')[1])
+                    self._price_decimals.add(decimals)
+
+        self._last_close = candle.close
+
+    def _collect_volume_data(self, candle: OHLCV) -> None:
+        """
+        Collect volume decimal statistics for quantity step analysis during writing.
+
+        Must run on the original float64 volumes: the file stores float32,
+        whose ~7 significant digits destroy a small lot step on large volumes,
+        so the analysis cannot be redone from the file afterwards.
+        """
+        volume = candle.volume
+        if volume <= 0.0:  # empty bars and -1.0 gap fills carry no quantity information
+            return
+        vol_str = str(volume)  # shortest round-trip repr, no float dust
+        if 'e' in vol_str or 'E' in vol_str:
+            vol_str = f"{volume:.20f}".rstrip('0')
+        decimals = len(vol_str.split('.')[1].rstrip('0')) if '.' in vol_str else 0
+        if decimals > _QTY_STEP_MAX_DECIMALS:
+            self._volume_dust_count += 1
+            return
+        self._volume_count += 1
+        if decimals > self._volume_max_decimals:
+            self._volume_max_decimals = decimals
+
+    def _analyze_tick_size(self) -> None:
+        """
+        Analyze collected price data to determine tick size using multiple methods.
+        """
+        if not self._price_changes:
+            # No data, use defaults
+            self._analyzed_tick_size = 0.01
+            self._analyzed_price_scale = 100
+            self._analyzed_min_move = 1
+            self._confidence = 0.1
+            return
+
+        # Try histogram-based method first for better noise handling
+        histogram_tick = self._calculate_histogram_tick()
+
+        if histogram_tick[0] > 0 and histogram_tick[1] > 0.7:
+            # High confidence histogram result, use it directly
+            self._analyzed_tick_size = histogram_tick[0]
+            self._analyzed_price_scale = int(round(1.0 / histogram_tick[0]))
+            self._analyzed_min_move = 1
+            self._confidence = histogram_tick[1]
+            return
+
+        # Fall back to other methods
+        # Method 1: Most frequent small change
+        freq_tick = self._calculate_frequency_tick()
+
+        # Method 2: Decimal places analysis
+        decimal_tick = self._calculate_decimal_tick()
+
+        # Combine methods with weighted confidence (no GCD)
+        tick_size, confidence = self._combine_tick_estimates(freq_tick, decimal_tick)
+
+        # Calculate price scale and min move
+        if tick_size > 0:
+            self._analyzed_tick_size = tick_size
+            self._analyzed_price_scale = int(round(1.0 / tick_size))
+            self._analyzed_min_move = 1
+            self._confidence = confidence
+        else:
+            # Fallback to defaults
+            self._analyzed_tick_size = 0.01
+            self._analyzed_price_scale = 100
+            self._analyzed_min_move = 1
+            self._confidence = 0.1
+
+    def _calculate_frequency_tick(self) -> tuple[float, float]:
+        """
+        Calculate tick size based on most frequent small changes.
+        Returns (tick_size, confidence)
+        """
+        if len(self._price_changes) < 10:
+            return 0, 0
+
+        # Apply float32 filtering first
+        filtered_changes = []
+        for c in self._price_changes[:100]:
+            if c > 0:
+                # Convert to float32 and back
+                float32_val = struct.unpack('f', struct.pack('f', c))[0]
+                # Round to reasonable precision for float32
+                rounded = round(float32_val, 6)
+                if rounded > 0:
+                    filtered_changes.append(rounded)
+
+        if len(filtered_changes) < 5:
+            return 0, 0
+
+        # Find most frequent changes
+        counter = Counter(filtered_changes)
+        most_common = counter.most_common(10)
+
+        if not most_common:
+            return 0, 0
+
+        # Find GCD of frequent changes to get base tick
+        frequent_changes = [change for change, count in most_common if count >= 2]
+        if len(frequent_changes) >= 2:
+            # Convert to integers for GCD
+            scale = 1000000  # 6 decimal places
+            int_changes = [int(round(c * scale)) for c in frequent_changes]
+
+            # Calculate GCD
+            result = int_changes[0]
+            for val in int_changes[1:]:
+                result = math_gcd(result, val)
+
+            tick_size = result / scale
+
+            # Confidence based on how many changes match this tick
+            matches = sum(1 for c in filtered_changes
+                          if abs(round(c / tick_size) * tick_size - c) < tick_size * 0.1)
+            confidence = min(matches / len(filtered_changes), 1.0)
+            return tick_size, confidence * 0.7  # Medium weight
+
+        return 0, 0
+
+    def _calculate_histogram_tick(self) -> tuple[float, float]:
+        """
+        Calculate tick size using histogram-based clustering approach.
+        This method is robust to float32 noise.
+        Returns (tick_size, confidence)
+        """
+        if len(self._price_changes) < 10:
+            return 0, 0
+
+        # Common tick sizes to test (from 1 to 0.00001)
+        candidate_ticks = [
+            1.0, 0.5, 0.25, 0.1, 0.05, 0.01, 0.005, 0.001,
+            0.0005, 0.0001, 0.00005, 0.00001, 0.000001
+        ]
+
+        best_tick = 0
+        best_score = 0
+
+        # Filter out zero changes and convert to float32 precision
+        changes = []
+        for change in self._price_changes[:200]:  # Use more samples for histogram
+            if change > 0:
+                # Round to float32 precision
+                float32_val = struct.unpack('f', struct.pack('f', change))[0]
+                changes.append(float32_val)
+
+        if len(changes) < 5:
+            return 0, 0
+
+        # Get min non-zero change to establish scale
+        min_change = min(changes)
+        avg_change = sum(changes) / len(changes)
+
+        for tick in candidate_ticks:
+            # Skip ticks that are too small (less than 1/10 of smallest change)
+            if tick < min_change * 0.1:
+                continue
+
+            # Skip ticks that are way too large
+            if tick > avg_change * 10:
+                continue
+
+            # Round all changes to this tick size
+            rounded = [round(c / tick) * tick for c in changes]
+
+            # Calculate how well the rounding fits
+            errors = [abs(c - r) for c, r in zip(changes, rounded)]
+            max_error = max(errors)
+
+            # Key insight: if max error is less than tick/2, this tick captures the grid well
+            if max_error < tick * 0.5:
+                # Count how many changes are multiples of this tick (within tolerance)
+                tolerance = tick * 0.1
+                multiples = sum(1 for c in changes if abs(round(c / tick) * tick - c) < tolerance)
+                multiple_ratio = multiples / len(changes)
+
+                # Score based on how many values are clean multiples
+                if multiple_ratio > 0.7:  # Most values are clean multiples
+                    score = multiple_ratio
+
+                    # Prefer larger ticks (less precision) when scores are similar
+                    # This helps choose 0.00001 over 0.000001 when both fit
+                    score *= (1.0 + tick * 100)  # Small bonus for larger ticks
+
+                    if score > best_score:
+                        best_score = score
+                        best_tick = tick
+
+        # If no good tick found with strict criteria, fall back to simple analysis
+        if best_tick == 0:
+            # Find the most common order of magnitude in changes
+            magnitudes = []
+            for c in changes:
+                if c > 0:
+                    # Find order of magnitude
+                    mag = 10 ** math.floor(math.log10(c))
+                    magnitudes.append(mag)
+
+            if magnitudes:
+                # Most common magnitude
+                counter = Counter(magnitudes)
+                common_mag = counter.most_common(1)[0][0]
+                # Use tick as 1/10 of common magnitude
+                best_tick = common_mag / 10
+                best_score = 0.5
+
+        # Calculate confidence based on score
+        if best_score > 0.8:
+            confidence = 0.9
+        elif best_score > 0.6:
+            confidence = 0.7
+        else:
+            confidence = best_score
+
+        return best_tick, confidence
+
+    def _calculate_decimal_tick(self) -> tuple[float, float]:
+        """
+        Calculate tick size based on decimal places.
+        Returns (tick_size, confidence)
+        """
+        if not self._price_decimals:
+            # No decimals found, probably integer prices
+            return 1.0, 0.5
+
+        # Filter out noise from float representation
+        # If we have 15 decimals, it's likely float noise
+        valid_decimals = [d for d in self._price_decimals if d <= 10]
+
+        if not valid_decimals:
+            # All decimals are noise, assume 2 decimal places (cents)
+            return 0.01, 0.3
+
+        # Use most common valid decimal places
+        max_decimals = max(valid_decimals)
+        tick_size = 10 ** (-max_decimals)
+
+        # Lower confidence for decimal-only method
+        return tick_size, 0.5
+
+    @staticmethod
+    def _combine_tick_estimates(freq: tuple[float, float],
+                                decimal: tuple[float, float]) -> tuple[float, float]:
+        """
+        Combine tick size estimates from frequency and decimal methods only.
+        Returns (tick_size, confidence)
+        """
+        estimates = []
+
+        if freq[0] > 0 and freq[1] > 0:
+            estimates.append(freq)
+        if decimal[0] > 0 and decimal[1] > 0:
+            estimates.append(decimal)
+
+        if not estimates:
+            return 0.01, 0.1  # Default fallback
+
+        # Use highest confidence estimate
+        best = max(estimates, key=lambda x: x[1])
+        return best
+
+    def _collect_trading_hours(self, candle: OHLCV) -> None:
+        """
+        Collect trading hours data from timestamps.
+        Only collect for candles with actual volume (not gaps).
+        """
+        if candle.volume <= 0:
+            return  # Skip gaps
+
+        # Convert timestamp to datetime
+        dt = datetime.fromtimestamp(candle.timestamp, tz=None)  # Local time
+
+        # Get weekday (1=Monday, 7=Sunday) and hour
+        weekday = dt.isoweekday()
+        hour = dt.hour
+
+        # Count occurrences
+        key = (weekday, hour)
+        self._trading_hours[key] = self._trading_hours.get(key, 0) + 1
+
+    def _collect_existing_trading_hours(self) -> None:
+        """
+        Collect trading hours data from existing file for opening hours analysis.
+        Only samples a subset of data for performance reasons.
+        """
+        if not self._file or self._size == 0:
+            return
+
+        # Save current position
+        current_pos = self._file.tell()
+
+        try:
+            # Sample data: read every Nth record for performance
+            # For large files, we don't need to read everything
+            sample_interval = max(1, self._size // 1000)  # Sample up to 1000 points
+
+            for i in range(0, self._size, sample_interval):
+                self._file.seek(i * RECORD_SIZE)
+                data = self._file.read(RECORD_SIZE)
+
+                if len(data) == RECORD_SIZE:
+                    # Unpack the record
+                    timestamp, open_val, high, low, close, volume = \
+                        struct.unpack('Ifffff', data)
+
+                    # Only collect if volume > 0 (real trading)
+                    if volume > 0:
+                        dt = datetime.fromtimestamp(timestamp, tz=None)
+                        weekday = dt.isoweekday()
+                        hour = dt.hour
+                        key = (weekday, hour)
+                        self._trading_hours[key] = self._trading_hours.get(key, 0) + 1
+
+        finally:
+            # Restore file position
+            self._file.seek(current_pos)
+
+    def _has_enough_data_for_opening_hours(self) -> bool:
+        """
+        Check if we have enough data to analyze opening hours based on timeframe.
+        """
+        if not self._trading_hours or not self._interval:
+            return False
+
+        # For daily or larger timeframes
+        if self._interval >= 86400:  # >= 1 day
+            # We need at least a few days to see a pattern
+            unique_days = len(set(day for day, hour in self._trading_hours.keys()))
+            return unique_days >= 3  # At least 3 different days
+
+        # For intraday timeframes
+        # Check if we have at least some meaningful data
+        # We need enough to see a pattern
+        data_points = sum(self._trading_hours.values())
+        points_per_hour = 3600 / self._interval
+        hours_covered = data_points / points_per_hour
+
+        # Need at least 2 hours of data to detect any pattern
+        # This allows even short sessions to be analyzed
+        return hours_covered >= 2
+
+    def _analyze_opening_hours(self) -> None:
+        """
+        Analyze collected trading hours to determine opening hours pattern.
+        Works for both intraday and daily timeframes.
+        """
+        if not self._trading_hours:
+            self._analyzed_opening_hours = None
+            return
+
+        # For daily or larger timeframes, analyze which days have trading
+        if self._interval and self._interval >= 86400:  # >= 1 day
+            hours: list = []
+            days_with_trading = set(day for day, hour in self._trading_hours.keys())
+
+            # Check if it's 24/7 (all 7 days have trading)
+            if len(days_with_trading) == 7:
+                # 24/7 trading pattern
+                for day in range(1, 8):
+                    hours.append(SymInfoInterval(
+                        day=day,
+                        start=time(0, 0, 0),
+                        end=time(23, 59, 59)
+                    ))
+            elif days_with_trading <= {1, 2, 3, 4, 5}:  # Monday-Friday only
+                # Business days pattern (stock/forex)
+                for day in range(1, 6):
+                    hours.append(SymInfoInterval(
+                        day=day,
+                        start=time(9, 30, 0),  # Default to US market hours
+                        end=time(16, 0, 0)
+                    ))
+            else:
+                # Mixed pattern - include all days that have trading
+                for day in sorted(days_with_trading):
+                    hours.append(SymInfoInterval(
+                        day=day,
+                        start=time(0, 0, 0),  # Default to full day for daily data
+                        end=time(23, 59, 59)
+                    ))
+            self._analyzed_opening_hours = hours
+            return
+
+        # For intraday data, analyze hourly patterns
+        # Check if it's 24/7 trading (crypto pattern)
+        total_hours = len(self._trading_hours)
+        if total_hours >= 168 * 0.7:  # 70% of all hours in a week (lowered threshold)
+            # Check if all hours have similar activity
+            counts = list(self._trading_hours.values())
+            avg_count = sum(counts) / len(counts)
+            variance = sum((c - avg_count) ** 2 for c in counts) / len(counts)
+
+            # If low variance, it's likely 24/7
+            if variance < avg_count * 0.5:
+                hours = []
+                for day in range(1, 8):
+                    hours.append(SymInfoInterval(
+                        day=day,
+                        start=time(0, 0, 0),
+                        end=time(23, 59, 59)
+                    ))
+                self._analyzed_opening_hours = hours
+                return
+
+        # Analyze per-day patterns for intraday
+        hours = []
+
+        for day in range(1, 8):  # Monday to Sunday
+            # Get all hours for this day
+            day_hours = [(hour, count) for (d, hour), count in self._trading_hours.items() if d == day]
+
+            if not day_hours:
+                continue  # No trading on this day
+
+            # Sort by hour
+            day_hours.sort(key=lambda x: x[0])
+
+            # Find continuous trading periods
+            periods = []
+            current_start = None
+            current_end = None
+
+            # Threshold: consider an hour active if it has at least 20% of average activity
+            total_count = sum(count for _, count in day_hours)
+            if total_count == 0:
+                continue
+            avg_hour_count = total_count / len(day_hours)
+            threshold = avg_hour_count * 0.2
+
+            for hour, count in day_hours:
+                if count >= threshold:
+                    if current_start is None:
+                        current_start = hour
+                        current_end = hour
+                    else:
+                        current_end = hour
+                else:
+                    if current_start is not None:
+                        periods.append((current_start, current_end))
+                        current_start = None
+                        current_end = None
+
+            # Add last period if exists
+            if current_start is not None:
+                periods.append((current_start, current_end))
+
+            # Convert periods to SymInfoInterval
+            for start_hour, end_hour in periods:
+                hours.append(SymInfoInterval(
+                    day=day,
+                    start=time(start_hour, 0, 0),
+                    end=time(end_hour, 59, 59)
+                ))
+
+        # If no opening hours detected, default to business hours
+        if not hours:
+            for day in range(1, 6):  # Monday to Friday
+                hours.append(SymInfoInterval(
+                    day=day,
+                    start=time(9, 30, 0),
+                    end=time(16, 0, 0)
+                ))
+
+        self._analyzed_opening_hours = hours
+
+    def _rebuild_with_correct_interval(self, new_interval: int) -> None:
+        """
+        Rebuild the entire file with the correct interval when a smaller interval is detected.
+        This happens when initial interval was wrong due to gaps.
+
+        :param new_interval: The correct interval to use
+        """
+        import tempfile
+        import shutil
+
+        if not self._file or self._size == 0:
+            return
+
+        # Save current file position and data
+        current_records = []
+
+        # Read all existing records
+        self._file.seek(0)
+        for i in range(self._size):
+            offset = i * RECORD_SIZE
+            self._file.seek(offset)
+            data = self._file.read(RECORD_SIZE)
+            if len(data) == RECORD_SIZE:
+                record = struct.unpack(STRUCT_FORMAT, data)
+                current_records.append(OHLCV(*record, extra_fields={}))
+
+        # Create temp file for rebuilding
+        temp_fd, temp_path = tempfile.mkstemp(suffix='.ohlcv.tmp', dir=os.path.dirname(self.path))
+        try:
+            # Close temp file descriptor as we'll open it differently
+            os.close(temp_fd)
+
+            # Create new writer with temp file
+            with OHLCVWriter(temp_path) as temp_writer:
+                # Write all records with correct interval
+                # The writer will now properly handle gaps
+                for record in current_records:
+                    temp_writer.write(record)
+
+            # Close current file and clean up extra CSV (will be rebuilt)
+            self._file.close()
+            self._close_extra_csv()
+            extra_path = Path(self.path).with_suffix('.extra.csv')
+            if extra_path.exists():
+                extra_path.unlink()
+            self._extra_headers = None
+            self._extra_row_count = 0
+
+            # Replace original with rebuilt file
+            shutil.move(temp_path, self.path)
+
+            # Reopen the file
+            f = open(self.path, 'rb+')
+            self._file = f
+            self._size = os.path.getsize(self.path) // RECORD_SIZE
+
+            # Reset interval to the correct one
+            self._interval = new_interval
+
+            # Position at end for appending
+            f.seek(0, os.SEEK_END)
+            self._current_pos = self._size
+
+            # Update last timestamp
+            if self._size > 0:
+                f.seek((self._size - 1) * RECORD_SIZE)
+                data = f.read(4)
+                self._last_timestamp = struct.unpack('I', data)[0]
+                f.seek(0, os.SEEK_END)
+
+        except Exception as e:
+            # Clean up temp file on error
+            if os.path.exists(temp_path):
+                try:
+                    os.unlink(temp_path)
+                except OSError:
+                    pass
+            raise IOError(f"Failed to rebuild file with correct interval: {e}")
+
+    def _parse_and_write_ohlcv_row(self, ts_str: str, row: list[str],
+                                   o_idx: int, h_idx: int, l_idx: int, c_idx: int, v_idx: int,
+                                   timestamp_format: str | None,
+                                   timezone: dt_timezone | ZoneInfo | None) -> None:
+        """
+        Parse timestamp and write OHLCV row with error handling.
+
+        :param ts_str: Timestamp string to parse
+        :param row: Data row containing OHLCV values
+        :param o_idx: Index of open price
+        :param h_idx: Index of high price
+        :param l_idx: Index of low price
+        :param c_idx: Index of close price
+        :param v_idx: Index of volume
+        :param timestamp_format: Optional timestamp format
+        :param timezone: Timezone object
+        :raises ValueError: If parsing or data conversion fails
+        """
+        # Parse timestamp
+        try:
+            timestamp = _parse_timestamp(ts_str, timestamp_format, timezone)
+        except Exception as e:
+            raise ValueError(f"Failed to parse timestamp '{ts_str}': {e}")
+
+        # Write OHLCV data
+        try:
+            self.write(OHLCV(
+                timestamp,
+                float(row[o_idx]),
+                float(row[h_idx]),
+                float(row[l_idx]),
+                float(row[c_idx]),
+                float(row[v_idx])
+            ))
+        except (ValueError, IndexError) as e:
+            raise ValueError(f"Invalid data in row: {e}")
+
+    def load_from_csv(self, path: str | Path,
+                      timestamp_format: str | None = None,
+                      timestamp_column: str | None = None,
+                      date_column: str | None = None,
+                      time_column: str | None = None,
+                      tz: str | None = None) -> None:
+        """
+        Load OHLCV data from CSV file using only builtin modules.
+
+        :param path: Path to CSV file
+        :param timestamp_format: Optional datetime fmt for parsing
+        :param timestamp_column: Column name for timestamp (default tries: timestamp, time, date)
+        :param date_column: When timestamp is split into date+time columns, date column name
+        :param time_column: When timestamp is split into date+time columns, time column name
+        :param tz: Timezone name (e.g. 'UTC', 'Europe/London', '+0100') for timestamp conversion
+        """
+        timezone = _parse_timezone_param(tz)
+
+        # Read CSV headers first
+        with open(path, 'r') as f:
+            reader = csv.reader(f)
+            headers = [h.lower() for h in next(reader)]  # Case insensitive
+
+            # Find timestamp and OHLCV columns
+            timestamp_idx, date_idx, time_idx = _find_timestamp_columns(headers, timestamp_column, date_column,
+                                                                        time_column)
+            o_idx, h_idx, l_idx, c_idx, v_idx = _find_ohlcv_columns(headers)
+
+            # Process data rows
+            for row in reader:
+                # Handle timestamp
+                if date_idx is not None and time_idx is not None:
+                    # Combine date and time
+                    ts_str = f"{row[date_idx]} {row[time_idx]}"
+                else:
+                    assert timestamp_idx is not None
+                    ts_str = row[timestamp_idx]
+
+                # Parse and write row
+                self._parse_and_write_ohlcv_row(ts_str, row, o_idx, h_idx, l_idx, c_idx, v_idx,
+                                                timestamp_format, timezone)
+
+    def load_from_txt(self, path: str | Path,
+                      timestamp_format: str | None = None,
+                      timestamp_column: str | None = None,
+                      date_column: str | None = None,
+                      time_column: str | None = None,
+                      tz: str | None = None) -> None:
+        """
+        Load OHLCV data from TXT file using only builtin modules.
+
+        :param path: Path to TXT file
+        :param timestamp_format: Optional datetime fmt for parsing
+        :param timestamp_column: Column name for timestamp (default tries: timestamp, time, date)
+        :param date_column: When timestamp is split into date+time columns, date column name
+        :param time_column: When timestamp is split into date+time columns, time column name
+        :param tz: Timezone name (e.g. 'UTC', 'Europe/London', '+0100') for timestamp conversion
+        """
+        timezone = _parse_timezone_param(tz)
+
+        # Auto-detect delimiter
+        with open(path, 'r') as f:
+            first_line = f.readline().strip()
+            if not first_line:
+                raise ValueError("File is empty or first line is blank")
+
+            # Check for common delimiters in order of preference
+            delimiters = ['\t', ';', '|']
+            delimiter_counts = {}
+
+            for delim in delimiters:
+                count = first_line.count(delim)
+                if count > 0:
+                    delimiter_counts[delim] = count
+
+            if not delimiter_counts:
+                raise ValueError("No supported delimiter found (tab, semicolon, or pipe)")
+
+            # Use delimiter with highest count
+            delimiter = max(delimiter_counts, key=lambda x: delimiter_counts[x])
+
+        # Read TXT file with manual parsing for better control
+        with open(path, 'r') as f:
+            lines = f.readlines()
+
+        if not lines:
+            raise ValueError("File is empty")
+
+        # Parse header line
+        header_line = lines[0].strip()
+        if not header_line:
+            raise ValueError("Header row is empty")
+
+        headers = self._parse_txt_line(header_line, delimiter)
+        headers = [h.lower().strip() for h in headers]  # Case insensitive
+
+        if not headers:
+            raise ValueError("No headers found")
+
+        # Find timestamp and OHLCV columns
+        timestamp_idx, date_idx, time_idx = _find_timestamp_columns(headers, timestamp_column, date_column, time_column)
+        o_idx, h_idx, l_idx, c_idx, v_idx = _find_ohlcv_columns(headers)
+
+        # Process data rows
+        for line in lines[1:]:  # Skip header
+            line = line.strip()
+            if not line:  # Skip empty lines
+                continue
+
+            row = self._parse_txt_line(line, delimiter)
+
+            if len(row) != len(headers):
+                raise ValueError(f"Row has {len(row)} columns, expected {len(headers)}")
+
+            # Strip whitespace from all fields
+            row = [field.strip() for field in row]
+
+            # Handle timestamp
+            if date_idx is not None and time_idx is not None:
+                # Combine date and time
+                ts_str = f"{row[date_idx]} {row[time_idx]}"
+            else:
+                ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else ""
+
+            # Parse and write row
+            self._parse_and_write_ohlcv_row(ts_str, row, o_idx, h_idx, l_idx, c_idx, v_idx,
+                                            timestamp_format, timezone)
+
+    @staticmethod
+    def _parse_txt_line(line: str, delimiter: str) -> list[str]:
+        """
+        Parse a single TXT line with proper handling of quoted fields and escape characters.
+
+        :param line: Line to parse
+        :param delimiter: Delimiter character
+        :return: List of parsed fields
+        :raises ValueError: If line format is invalid
+        """
+        if not line:
+            return []
+
+        fields = []
+        current_field = ""
+        in_quotes = False
+        quote_char = None
+        i = 0
+
+        while i < len(line):
+            char = line[i]
+
+            # Handle escape characters
+            if char == '\\' and i + 1 < len(line):
+                next_char = line[i + 1]
+                if next_char in ['"', "'", '\\', 'n', 't', 'r']:
+                    if next_char == 'n':
+                        current_field += '\n'
+                    elif next_char == 't':
+                        current_field += '\t'
+                    elif next_char == 'r':
+                        current_field += '\r'
+                    else:
+                        current_field += next_char
+                    i += 2
+                    continue
+                else:
+                    current_field += char
+                    i += 1
+                    continue
+
+            # Handle quotes
+            if char in ['"', "'"] and not in_quotes:
+                in_quotes = True
+                quote_char = char
+                i += 1
+                continue
+            elif char == quote_char and in_quotes:
+                # Check for escaped quote (double quote)
+                if i + 1 < len(line) and line[i + 1] == quote_char:
+                    current_field += char
+                    i += 2
+                    continue
+                else:
+                    in_quotes = False
+                    quote_char = None
+                    i += 1
+                    continue
+
+            # Handle delimiter
+            if char == delimiter and not in_quotes:
+                fields.append(current_field)
+                current_field = ""
+                i += 1
+                continue
+
+            # Regular character
+            current_field += char
+            i += 1
+
+        # Add the last field
+        fields.append(current_field)
+
+        # Validate that quotes are properly closed
+        if in_quotes:
+            raise ValueError(f"Unclosed quote in line: {line[:50]}...")
+
+        return fields
+
+    def load_from_json(self, path: str | Path,
+                       timestamp_format: str | None = None,
+                       timestamp_field: str | None = None,
+                       date_field: str | None = None,
+                       time_field: str | None = None,
+                       tz: str | None = None,
+                       mapping: dict[str, str] | None = None) -> None:
+        """
+        Load OHLCV data from JSON file using only builtin modules.
+
+        :param path: Path to JSON file
+        :param timestamp_format: Optional datetime format for parsing
+        :param timestamp_field: Field name for timestamp (default tries: timestamp, time, date, t)
+        :param date_field: When timestamp is split, date field name
+        :param time_field: When timestamp is split, time field name
+        :param tz: Timezone name (e.g. 'UTC', 'Europe/London', '+0100')
+        :param mapping: Optional field mapping, e.g. {'timestamp': 't', 'volume': 'vol'}
+        """
+        timezone = _parse_timezone_param(tz)
+
+        # Setup field mapping
+        field_mapping: dict[str, str] = mapping or {}
+        field_map = {
+            'timestamp': field_mapping.get('timestamp', timestamp_field),
+            'open': field_mapping.get('open', 'open'),
+            'high': field_mapping.get('high', 'high'),
+            'low': field_mapping.get('low', 'low'),
+            'close': field_mapping.get('close', 'close'),
+            'volume': field_mapping.get('volume', 'volume')
+        }
+
+        # Load JSON file
+        data = None
+        with open(path, 'r') as f:
+            data = json.load(f)
+
+        # Ensure we have a list of records
+        if isinstance(data, dict):
+            # Some APIs wrap the data in an object
+            for key in ['data', 'candles', 'ohlcv', 'results']:
+                if key in data and isinstance(data[key], list):
+                    data = data[key]
+                    break
+            else:
+                raise ValueError("Could not find OHLCV data array in JSON")
+
+        if not isinstance(data, list):
+            raise ValueError("JSON must contain an array of OHLCV records")
+
+        # Find timestamp field if not specified
+        if not field_map['timestamp'] and not (date_field and time_field):
+            common_names = ['timestamp', 'time', 'date', 't']
+            for record in data[:1]:  # Check just first record
+                for name in common_names:
+                    if name in record:
+                        field_map['timestamp'] = name
+                        break
+                if field_map['timestamp']:
+                    break
+            if not field_map['timestamp']:
+                raise ValueError("Could not find timestamp field")
+
+        # Process records
+        for record in data:
+            # Get timestamp
+            try:
+                if date_field and time_field:
+                    # Combine date and time
+                    ts_str = f"{record[date_field]} {record[time_field]}"
+                else:
+                    ts_str = str(record[field_map['timestamp']])
+
+                # Convert timestamp
+                timestamp = _parse_timestamp(ts_str, timestamp_format, timezone)
+
+                # Get OHLCV values
+                try:
+                    self.write(OHLCV(
+                        timestamp,
+                        float(record[field_map['open']]),
+                        float(record[field_map['high']]),
+                        float(record[field_map['low']]),
+                        float(record[field_map['close']]),
+                        float(record[field_map['volume']])
+                    ))
+                except KeyError as e:
+                    raise ValueError(f"Missing field in record: {e}")
+                except ValueError as e:
+                    raise ValueError(f"Invalid value in record: {e}")
+
+            except Exception as e:
+                raise ValueError(f"Failed to process record: {e}")
+
+
+class OHLCVReader:
+    """
+    Very fast OHLCV data reader using memory mapping.
+    """
+
+    __slots__ = ('path', '_file', '_mmap', '_size', '_start_timestamp', '_interval',
+                 '_extra_data', '_extra_headers')
+
+    def __init__(self, path: str | Path):
+        self.path = str(path)
+        self._file = None
+        self._mmap = None
+        self._size = 0
+        self._start_timestamp = None
+        self._interval = None
+        self._extra_data: list[dict[str, int | float | str]] | None = None
+        self._extra_headers: list[str] | None = None
+
+    def __enter__(self):
+        self.open()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    @property
+    def size(self) -> int:
+        """
+        Number of records in the file
+        """
+        return self._size
+
+    @property
+    def start_timestamp(self) -> int | None:
+        """
+        Timestamp of the first record
+        """
+        return self._start_timestamp
+
+    @property
+    def start_datetime(self) -> datetime:
+        """
+        Datetime of the first record
+        """
+        assert self._start_timestamp is not None
+        return datetime.fromtimestamp(self._start_timestamp, UTC)
+
+    @property
+    def end_timestamp(self) -> int | None:
+        """
+        Timestamp of the last record
+        """
+        if self._size == 0:
+            return None
+
+        # Read the actual timestamp from the last record instead of calculating it
+        # This is necessary because gap filling may create non-uniform intervals
+        if self._mmap and self._size > 0:
+            last_record_offset = (self._size - 1) * RECORD_SIZE
+            return struct.unpack('I', self._mmap[last_record_offset:last_record_offset + 4])[0]
+
+        return None
+
+    @property
+    def end_datetime(self) -> datetime:
+        """
+        Datetime of the last record
+        """
+        ts = self.end_timestamp
+        assert ts is not None
+        return datetime.fromtimestamp(ts, UTC)
+
+    @property
+    def interval(self) -> int | None:
+        """
+        Interval between records
+        """
+        return self._interval
+
+    def open(self) -> 'OHLCVReader':
+        """
+        Open file and create memory mapping
+        """
+        self._file = open(self.path, 'rb')
+        if os.path.getsize(self.path) > 0:
+            # Detect if this is a text file masquerading as binary OHLCV
+            self._file.seek(0)
+            first_chunk = self._file.read(32)
+            self._file.seek(0)  # Reset position
+
+            try:
+                # If 256 bytes decode as ASCII, it's definitely not binary OHLCV
+                first_chunk.decode('ascii')
+
+                # If we get here, it's text - show error with CLI fix
+                raise ValueError(
+                    f"Text file detected with .ohlcv extension!\n"
+                    f"To convert CSV to binary OHLCV format:\n"
+                    f"  pyne data convert-from {Path(self.path).with_suffix('.csv')} "
+                    f"--symbol YOUR_SYMBOL --provider custom"
+                )
+            except UnicodeDecodeError:
+                # Can't decode as ASCII → it's binary, proceed normally
+                pass
+
+            self._mmap = mmap.mmap(self._file.fileno(), 0, access=mmap.ACCESS_READ)
+            self._size = os.path.getsize(self.path) // RECORD_SIZE
+
+            if self._size >= 2:
+                self._start_timestamp = struct.unpack('I', self._mmap[0:4])[0]
+                second_timestamp = struct.unpack('I', self._mmap[RECORD_SIZE:RECORD_SIZE + 4])[0]
+                self._interval = second_timestamp - self._start_timestamp
+
+        self._load_extra_csv()
+
+        return self
+
+    def _load_extra_csv(self) -> None:
+        """
+        Load extra fields from sidecar .extra.csv file if it exists.
+        The sidecar is position-aligned with the binary OHLCV file.
+        """
+        extra_path = Path(self.path).with_suffix('.extra.csv')
+        if not extra_path.exists():
+            return
+
+        with open(extra_path, 'r', newline='') as f:
+            reader = csv.reader(f)
+            headers = next(reader, None)
+            if not headers:
+                return
+
+            self._extra_headers = headers
+
+            # Detect column types from first non-empty data row
+            rows_raw: list[list[str]] = []
+            col_is_numeric: list[bool | None] = [None] * len(headers)
+
+            for row in reader:
+                rows_raw.append(row)
+                for i, val in enumerate(row):
+                    if col_is_numeric[i] is None and val and val.lower() not in ('', 'nan', 'na'):
+                        try:
+                            float(val)
+                            col_is_numeric[i] = True
+                        except ValueError:
+                            col_is_numeric[i] = False
+
+            # Default undetected columns to string
+            col_is_numeric = [v if v is not None else False for v in col_is_numeric]
+
+            # Parse all rows with detected types
+            extra_data: list[dict[str, int | float | str]] = []
+            for row in rows_raw:
+                parsed: dict[str, int | float | str] = {}
+                for i, header in enumerate(headers):
+                    val = row[i] if i < len(row) else ''
+                    if col_is_numeric[i]:
+                        if not val or val.lower() in ('nan', 'na', ''):
+                            parsed[header] = float('nan')
+                        else:
+                            parsed[header] = float(val)
+                    else:
+                        parsed[header] = val
+                extra_data.append(parsed)
+            self._extra_data = extra_data
+
+    def __iter__(self) -> Iterator[OHLCV]:
+        """
+        Iterate through all candles
+        """
+        for pos in range(self._size):
+            yield self.read(pos)
+
+    def read(self, position: int) -> OHLCV:
+        """
+        Read a single candle at given position
+        """
+        if position < 0 or position >= self._size:
+            raise IndexError("Position out of range")
+
+        assert self._mmap is not None
+
+        offset = position * RECORD_SIZE
+        data = struct.unpack(STRUCT_FORMAT, self._mmap[offset:offset + RECORD_SIZE])
+
+        extra = {}
+        if self._extra_data is not None and position < len(self._extra_data):
+            extra = self._extra_data[position]
+
+        return OHLCV(*data, extra_fields=extra)
+
+    def read_from(self, start_timestamp: int, end_timestamp: int | None = None, skip_gaps: bool = True) \
+            -> Iterator[OHLCV]:
+        """
+        Read bars starting from timestamp, using direct position calculation.
+
+        :param start_timestamp: Start timestamp
+        :param end_timestamp: End timestamp, if None, read until the end
+        :param skip_gaps: Skip gaps in data, the writer fill gaps with the last value with -1 volume,
+                          this will skip them (default)
+        :raises ValueError: If start_timestamp is after the last bar
+        """
+        if not self._size or not self._interval:
+            return
+
+        # Calculate start and end positions
+        start_pos, end_pos = self.get_positions(start_timestamp, end_timestamp)
+
+        # Yield the calculated range
+        for pos in range(start_pos, end_pos):
+            ohlcv = self.read(pos)
+            # Skip gaps if needed
+            if skip_gaps and ohlcv.volume < 0:
+                continue
+            yield ohlcv
+
+    def close(self):
+        """
+        Close file and memory mapping
+        """
+        if self._mmap:
+            self._mmap.close()
+            self._mmap = None
+        if self._file:
+            self._file.close()
+            self._file = None
+        self._extra_data = None
+        self._extra_headers = None
+
+    def get_positions(self, start_timestamp: int | None = None, end_timestamp: int | None = None) -> tuple[int, int]:
+        """
+        Get start and end positions for given timestamps
+
+        :param start_timestamp: Start timestamp
+        :param end_timestamp: End timestamp
+        :return: Tuple of start and end positions
+        """
+        if not self._size or not self._interval:
+            return 0, 0
+        assert self._start_timestamp is not None
+
+        # Calculate start position
+        if start_timestamp is None:
+            start_pos = 0
+        else:
+            start_diff = start_timestamp - self._start_timestamp
+            if start_diff < 0:
+                start_pos = 0
+            else:
+                start_pos = min(start_diff // self._interval, self._size - 1)
+
+        # Calculate end position if provided
+        if end_timestamp is None:
+            end_pos = self._size
+        else:
+            # If end_timestamp >= actual last record timestamp, use full size
+            # This handles gap-filled files where end_timestamp doesn't align with interval
+            actual_end_ts = self.end_timestamp
+            if actual_end_ts and end_timestamp >= actual_end_ts:
+                end_pos = self._size
+            else:
+                end_diff = end_timestamp - self._start_timestamp
+                end_pos = min(end_diff // self._interval + 1, self._size)
+
+        return start_pos, end_pos
+
+    def get_size(self, start_timestamp: int | None = None, end_timestamp: int | None = None) -> int:
+        """
+        Get number of records between timestamps
+
+        :param start_timestamp: Start timestamp
+        :param end_timestamp: End timestamp
+        :return: Number of records
+        """
+        if not self._size or not self._interval:
+            return 0
+
+        start_pos, end_pos = self.get_positions(start_timestamp, end_timestamp)
+        return end_pos - start_pos
+
+    def save_to_csv(self, path: str, as_datetime=False) -> None:
+        """
+        Save OHLCV data to CSV file
+
+        :param path: Path to the CSV file
+        :param as_datetime: Save timestamp as datetime string
+        """
+
+        with open(path, 'w') as f:
+            if as_datetime:
+                f.write('time,open,high,low,close,volume\n')
+            else:
+                f.write('timestamp,open,high,low,close,volume\n')
+            for candle in self:
+                # Skip gaps (volume == -1)
+                if candle.volume == -1:
+                    continue
+                if as_datetime:
+                    f.write(f"{datetime.fromtimestamp(candle.timestamp, UTC)},{_format_float(candle.open)},"
+                            f"{_format_float(candle.high)},{_format_float(candle.low)},{_format_float(candle.close)},"
+                            f"{_format_float(candle.volume)}\n")
+                else:
+                    f.write(f"{candle.timestamp},{_format_float(candle.open)},{_format_float(candle.high)},"
+                            f"{_format_float(candle.low)},{_format_float(candle.close)},"
+                            f"{_format_float(candle.volume)}\n")
+
+    def save_to_json(self, path: str, as_datetime: bool = False) -> None:
+        """
+        Save OHLCV data to JSON file.
+
+        The output fmt is either:
+        [
+            {
+                "timestamp": 1234567890,  // or "time": "2024-01-07 12:34:56+00:00" if as_datetime is True
+                "open": 100.0,
+                "high": 101.0,
+                "low": 99.0,
+                "close": 100.5,
+                "volume": 1000.0
+            },
+            ...
+        ]
+
+        :param path: Path to save the JSON file
+        :param as_datetime: If True, convert timestamps to ISO fmt datetime strings
+        """
+        data = []
+        for candle in self:
+            # Skip gaps (volume == -1)
+            if candle.volume == -1:
+                continue
+            if as_datetime:
+                item = {
+                    "time": datetime.fromtimestamp(candle.timestamp, UTC).isoformat(),
+                    "open": _format_float(candle.open),
+                    "high": _format_float(candle.high),
+                    "low": _format_float(candle.low),
+                    "close": _format_float(candle.close),
+                    "volume": _format_float(candle.volume)
+                }
+            else:
+                item = {
+                    "timestamp": candle.timestamp,
+                    "open": _format_float(candle.open),
+                    "high": _format_float(candle.high),
+                    "low": _format_float(candle.low),
+                    "close": _format_float(candle.close),
+                    "volume": _format_float(candle.volume)
+                }
+            data.append(item)
+
+        with open(path, 'w') as f:
+            json.dump(data, f, indent=2)  # Use indent for human-readable fmt  # noqa