vizflow 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

vizflow/__init__.py

@@ -5,13 +5,11 @@ Usage:
     import vizflow as vf
 """
 
-__version__ = "0.5.0"
+__version__ = "0.5.2"
 
-from .config import ColumnSchema, Config, get_config, set_config
+from .config import Config, get_config, set_config
 from .io import (
-    load_alpha,
     load_calendar,
-    load_trade,
     scan_alpha,
     scan_alphas,
     scan_trade,
@@ -19,4 +17,11 @@ from .io import (
 )
 from .market import CN, CRYPTO, Market, Session
 from .ops import aggregate, bin, forward_return, parse_time
-from .presets import JYAO_V20251114, PRESETS, YLIN_V20251204
+from .schema_evolution import (
+    JYAO_V20251114,
+    SCHEMAS,
+    YLIN_V20251204,
+    ColumnSpec,
+    SchemaEvolution,
+    get_schema,
+)

vizflow/config.py

@@ -4,25 +4,28 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .schema_evolution import SchemaEvolution
 
 # Global config instance
 _global_config: Config | None = None
 
 
-@dataclass
-class ColumnSchema:
-    """Schema for a column with type casting.
+def _validate_date(date: str) -> None:
+    """Validate date string format to prevent path traversal.
 
-    Attributes:
-        cast_to: Target type after casting (e.g. pl.Int64)
+    Args:
+        date: Date string to validate
 
-    Example:
-        # Handle float precision errors: 1.00000002 → 1
-        ColumnSchema(cast_to=pl.Int64)
+    Raises:
+        ValueError: If date is not exactly 8 digits (YYYYMMDD format)
     """
-
-    cast_to: Any  # pl.DataType, but avoid import for now
+    if not (len(date) == 8 and date.isdigit()):
+        raise ValueError(
+            f"Invalid date format: {date!r}. Expected YYYYMMDD (8 digits)."
+        )
 
 
 @dataclass
@@ -38,14 +41,20 @@ class Config:
         replay_dir: Directory for FIFO replay output (materialization 1)
         aggregate_dir: Directory for aggregation output (materialization 2)
         market: Market identifier, e.g. "CN"
-        alpha_columns: Mapping from semantic names to alpha column names
-        trade_columns: Mapping from semantic names to trade column names
-        alpha_schema: Schema evolution for alpha columns
-        trade_schema: Schema evolution for trade columns
+        trade_schema: Schema evolution for trade data (name or SchemaEvolution)
+        alpha_schema: Schema evolution for alpha data (name or SchemaEvolution)
         binwidths: Mapping from column names to bin widths
         group_by: Columns to group by in aggregation
         horizons: List of forward return horizons in seconds
         time_cutoff: Optional time cutoff (e.g. 143000000 for 14:30:00)
+
+    Example:
+        >>> config = vf.Config(
+        ...     trade_dir=Path("data/ylin/trade"),
+        ...     trade_pattern="{date}.meords",
+        ...     trade_schema="ylin_v20251204",  # Use registered schema by name
+        ...     market="CN",
+        ... )
     """
 
     # === Input Paths ===
@@ -62,17 +71,10 @@ class Config:
     # === Market ===
     market: str = "CN"
 
-    # === Column Mapping ===
-    alpha_columns: dict[str, str] = field(default_factory=dict)
-    trade_columns: dict[str, str] = field(default_factory=dict)
-
     # === Schema Evolution ===
-    alpha_schema: dict[str, ColumnSchema] = field(default_factory=dict)
-    trade_schema: dict[str, ColumnSchema] = field(default_factory=dict)
-
-    # === Column Mapping Presets ===
-    trade_preset: str | None = None  # "ylin" or None
-    alpha_preset: str | None = None  # "jyao_v20251114" or None
+    # Can be a string (schema name) or SchemaEvolution instance
+    trade_schema: str | SchemaEvolution | None = None
+    alpha_schema: str | SchemaEvolution | None = None
 
     # === Aggregation ===
     binwidths: dict[str, float] = field(default_factory=dict)
@@ -83,7 +85,11 @@ class Config:
     time_cutoff: int | None = None
 
     def __post_init__(self):
-        """Convert paths to Path objects if needed."""
+        """Convert string paths to Path objects.
+
+        Note: String values for path fields (alpha_dir, trade_dir, calendar_path,
+        replay_dir, aggregate_dir) are automatically converted to Path objects.
+        """
         if isinstance(self.alpha_dir, str):
             self.alpha_dir = Path(self.alpha_dir)
         if isinstance(self.trade_dir, str):
@@ -95,20 +101,6 @@ class Config:
         if isinstance(self.aggregate_dir, str):
             self.aggregate_dir = Path(self.aggregate_dir)
 
-    def col(self, semantic: str, source: str = "trade") -> str:
-        """Get actual column name from semantic name.
-
-        Args:
-            semantic: Semantic column name (e.g. "timestamp", "price")
-            source: "alpha" or "trade"
-
-        Returns:
-            Actual column name, or the semantic name if no mapping exists
-        """
-        if source == "alpha":
-            return self.alpha_columns.get(semantic, semantic)
-        return self.trade_columns.get(semantic, semantic)
-
     def get_alpha_path(self, date: str) -> Path:
         """Get alpha file path for a date.
 
@@ -119,8 +111,9 @@ class Config:
             Full path to alpha file
 
         Raises:
-            ValueError: If alpha_dir is not set
+            ValueError: If alpha_dir is not set or date format is invalid
         """
+        _validate_date(date)
         if self.alpha_dir is None:
             raise ValueError("alpha_dir is not set in Config")
         return self.alpha_dir / self.alpha_pattern.format(date=date)
@@ -135,8 +128,9 @@ class Config:
             Full path to trade file
 
         Raises:
-            ValueError: If trade_dir is not set
+            ValueError: If trade_dir is not set or date format is invalid
         """
+        _validate_date(date)
         if self.trade_dir is None:
             raise ValueError("trade_dir is not set in Config")
         return self.trade_dir / self.trade_pattern.format(date=date)
@@ -152,8 +146,9 @@ class Config:
             Full path to replay output file
 
         Raises:
-            ValueError: If replay_dir is not set
+            ValueError: If replay_dir is not set or date format is invalid
         """
+        _validate_date(date)
         if self.replay_dir is None:
             raise ValueError("replay_dir is not set in Config")
         return self.replay_dir / f"{date}{suffix}"
@@ -169,8 +164,9 @@ class Config:
             Full path to aggregate output file
 
         Raises:
-            ValueError: If aggregate_dir is not set
+            ValueError: If aggregate_dir is not set or date format is invalid
         """
+        _validate_date(date)
         if self.aggregate_dir is None:
             raise ValueError("aggregate_dir is not set in Config")
         return self.aggregate_dir / f"{date}{suffix}"

vizflow/io.py

@@ -2,110 +2,42 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
 import polars as pl
 
 from .config import Config, get_config
+from .schema_evolution import SchemaEvolution, get_schema
 
-if TYPE_CHECKING:
-    pass
-
-
-def load_alpha(date: str, config: Config | None = None) -> pl.LazyFrame:
-    """Load alpha data for a date with automatic schema evolution.
-
-    Args:
-        date: Date string, e.g. "20241001"
-        config: Config to use, or get_config() if None
-
-    Returns:
-        LazyFrame with schema evolution applied
-
-    Example:
-        >>> config = vf.Config(
-        ...     output_dir=Path("/data/output"),
-        ...     alpha_dir=Path("/data/alpha"),
-        ...     alpha_schema={"qty": vf.ColumnSchema(cast_to=pl.Int64)}
-        ... )
-        >>> vf.set_config(config)
-        >>> alpha = vf.load_alpha("20241001")
-    """
-    config = config or get_config()
-    path = config.get_alpha_path(date)
-    df = pl.scan_ipc(path)
-
-    # Apply schema evolution (type casting)
-    for col_name, schema in config.alpha_schema.items():
-        df = df.with_columns(pl.col(col_name).cast(schema.cast_to))
-
-    return df
-
-
-def load_trade(date: str, config: Config | None = None) -> pl.LazyFrame:
-    """Load trade data for a date with automatic schema evolution.
-
-    Args:
-        date: Date string, e.g. "20241001"
-        config: Config to use, or get_config() if None
-
-    Returns:
-        LazyFrame with schema evolution applied
-
-    Example:
-        >>> config = vf.Config(
-        ...     output_dir=Path("/data/output"),
-        ...     trade_dir=Path("/data/trade"),
-        ...     trade_schema={"qty": vf.ColumnSchema(cast_to=pl.Int64)}
-        ... )
-        >>> vf.set_config(config)
-        >>> trade = vf.load_trade("20241001")  # qty: 1.00000002 → 1
-    """
-    config = config or get_config()
-    path = config.get_trade_path(date)
-    df = pl.scan_ipc(path)
 
-    # Apply schema evolution (type casting)
-    for col_name, schema in config.trade_schema.items():
-        df = df.with_columns(pl.col(col_name).cast(schema.cast_to))
-
-    return df
-
-
-def load_calendar(config: Config | None = None) -> pl.DataFrame:
-    """Load trading calendar.
+def _resolve_schema(
+    schema_ref: str | SchemaEvolution | None,
+) -> SchemaEvolution | None:
+    """Resolve schema reference to SchemaEvolution instance.
 
     Args:
-        config: Config to use, or get_config() if None
+        schema_ref: Schema name string, SchemaEvolution instance, or None.
 
     Returns:
-        DataFrame with date, prev_date, next_date columns
-
-    Raises:
-        ValueError: If calendar_path is not set in config
-
-    Example:
-        >>> config = vf.Config(
-        ...     output_dir=Path("/data/output"),
-        ...     calendar_path=Path("/data/calendar.parquet")
-        ... )
-        >>> vf.set_config(config)
-        >>> calendar = vf.load_calendar()
+        SchemaEvolution instance or None.
     """
-    config = config or get_config()
-    if config.calendar_path is None:
-        raise ValueError("calendar_path is not set in Config")
-    return pl.read_parquet(config.calendar_path)
+    if schema_ref is None:
+        return None
+    if isinstance(schema_ref, SchemaEvolution):
+        return schema_ref
+    return get_schema(schema_ref)
 
 
-def _scan_file(path) -> pl.LazyFrame:
-    """Scan a file based on its extension.
+def _scan_file(
+    path,
+    schema: SchemaEvolution | None = None,
+) -> pl.LazyFrame:
+    """Scan a file based on its extension with optional schema.
 
     Args:
-        path: Path to file
+        path: Path to file.
+        schema: Optional SchemaEvolution for CSV parsing options.
 
     Returns:
-        LazyFrame from the file
+        LazyFrame from the file.
 
     Supported formats:
         - .feather, .ipc, .arrow: IPC format (pl.scan_ipc)
@@ -117,58 +49,114 @@ def _scan_file(path) -> pl.LazyFrame:
     if suffix in ("feather", "ipc", "arrow"):
         return pl.scan_ipc(path)
     elif suffix in ("csv", "meords"):
-        return pl.scan_csv(path)
+        csv_kwargs = {}
+        if schema:
+            schema_overrides = schema.get_schema_overrides()
+            if schema_overrides:
+                csv_kwargs["schema_overrides"] = schema_overrides
+            null_values = schema.get_null_values()
+            if null_values:
+                csv_kwargs["null_values"] = null_values
+        return pl.scan_csv(path, **csv_kwargs)
     elif suffix == "parquet":
         return pl.scan_parquet(path)
     else:
-        # Default to IPC
-        return pl.scan_ipc(path)
+        raise ValueError(
+            f"Unsupported file format: .{suffix}. "
+            "Supported: .feather, .ipc, .arrow, .csv, .meords, .parquet"
+        )
+
+
+def _apply_schema_evolution(
+    df: pl.LazyFrame,
+    schema: SchemaEvolution,
+) -> pl.LazyFrame:
+    """Apply full schema evolution: drop, rename, cast.
+
+    Args:
+        df: LazyFrame to transform.
+        schema: SchemaEvolution with transformation rules.
+
+    Returns:
+        Transformed LazyFrame.
+    """
+    existing = set(df.collect_schema().names())
+
+    # Step 1: Drop excluded columns
+    drop_cols = schema.get_drop_columns()
+    to_drop = [c for c in drop_cols if c in existing]
+    if to_drop:
+        df = df.drop(to_drop)
+        existing -= set(to_drop)
+
+    # Step 2: Rename columns
+    rename_map = schema.get_rename_map()
+    to_rename = {k: v for k, v in rename_map.items() if k in existing}
+    if to_rename:
+        df = df.rename(to_rename)
+        # Update existing names after rename
+        for old, new in to_rename.items():
+            existing.discard(old)
+            existing.add(new)
+
+    # Step 3: Cast columns (using FINAL names after rename)
+    cast_map = schema.get_cast_map()
+    for col_name, dtype in cast_map.items():
+        if col_name in existing:
+            df = df.with_columns(pl.col(col_name).cast(dtype))
+
+    return df
 
 
 def scan_trade(date: str, config: Config | None = None) -> pl.LazyFrame:
-    """Scan single date trade file with column mapping.
+    """Scan single date trade file with schema evolution.
 
-    Supports both IPC/feather format and CSV format (including .meords files).
+    Supports IPC/feather, CSV (including .meords), and Parquet formats.
 
     Args:
         date: Date string, e.g. "20241001"
         config: Config to use, or get_config() if None
 
     Returns:
-        LazyFrame with column mapping and schema evolution applied
+        LazyFrame with schema evolution applied
 
     Example:
         >>> config = vf.Config(
-        ...     trade_dir=Path("/data/yuanzhao/"),
+        ...     trade_dir=Path("/data/ylin/trade"),
         ...     trade_pattern="{date}.meords",
-        ...     trade_preset="ylin_v20251204",
+        ...     trade_schema="ylin_v20251204",
         ... )
         >>> vf.set_config(config)
         >>> df = vf.scan_trade("20241001")
     """
     config = config or get_config()
     path = config.get_trade_path(date)
-    df = _scan_file(path)
-    return _apply_trade_mapping(df, config)
+    schema = _resolve_schema(config.trade_schema)
+
+    df = _scan_file(path, schema=schema)
+    if schema:
+        df = _apply_schema_evolution(df, schema)
+
+    return df
 
 
 def scan_trades(config: Config | None = None) -> pl.LazyFrame:
-    """Scan all trade files with column mapping.
+    """Scan all trade files with schema evolution.
 
     Args:
         config: Config to use, or get_config() if None
 
     Returns:
-        LazyFrame with column mapping and schema evolution applied
+        LazyFrame with schema evolution applied
 
     Raises:
         ValueError: If trade_dir is not set or no files found
 
     Example:
         >>> config = vf.Config(
-        ...     trade_dir=Path("/data/yuanzhao/"),
-        ...     trade_pattern="{date}.feather",
-        ...     trade_preset="ylin_v20251204",
+        ...     trade_dir=Path("/data/ylin/trade"),
+        ...     trade_pattern="{date}.meords",
+        ...     trade_schema="ylin_v20251204",
         ... )
         >>> vf.set_config(config)
         >>> df = vf.scan_trades()
@@ -182,86 +170,56 @@ def scan_trades(config: Config | None = None) -> pl.LazyFrame:
     if not files:
         raise ValueError(f"No files found matching {pattern} in {config.trade_dir}")
 
+    schema = _resolve_schema(config.trade_schema)
+
     # Concatenate all files using lazy scanning
-    dfs = [_scan_file(f) for f in files]
+    dfs = [_scan_file(f, schema=schema) for f in files]
     df = pl.concat(dfs)
-    return _apply_trade_mapping(df, config)
-
-
-def _apply_trade_mapping(df: pl.LazyFrame, config: Config) -> pl.LazyFrame:
-    """Apply column rename + schema evolution for trade data."""
-    df = _apply_rename(df, config.trade_preset)
-    for col_name, schema in config.trade_schema.items():
-        df = df.with_columns(pl.col(col_name).cast(schema.cast_to))
-    return df
-
-
-def _apply_alpha_mapping(df: pl.LazyFrame, config: Config) -> pl.LazyFrame:
-    """Apply column rename + schema evolution for alpha data."""
-    df = _apply_rename(df, config.alpha_preset)
-    for col_name, schema in config.alpha_schema.items():
-        df = df.with_columns(pl.col(col_name).cast(schema.cast_to))
-    return df
-
-
-def _apply_rename(df: pl.LazyFrame, preset: str | None) -> pl.LazyFrame:
-    """Apply column rename from preset name.
-
-    Args:
-        df: LazyFrame to rename columns
-        preset: Preset name (e.g., "ylin", "jyao_v20251114") or None
-    """
-    # Drop record type prefix column if present (from CSV files)
-    existing = set(df.collect_schema().names())
-    if "#HFTORD" in existing:
-        df = df.drop("#HFTORD")
-        existing.remove("#HFTORD")
-
-    # Get rename map from preset
-    rename_map = _get_rename_map(preset)
 
-    if rename_map:
-        existing = set(df.collect_schema().names())
-        to_rename = {k: v for k, v in rename_map.items() if k in existing}
-        if to_rename:
-            df = df.rename(to_rename)
+    if schema:
+        df = _apply_schema_evolution(df, schema)
 
     return df
 
 
 def scan_alpha(date: str, config: Config | None = None) -> pl.LazyFrame:
-    """Scan single date alpha file with column mapping.
+    """Scan single date alpha file with schema evolution.
 
     Args:
         date: Date string, e.g. "20241001"
         config: Config to use, or get_config() if None
 
     Returns:
-        LazyFrame with column mapping and schema evolution applied
+        LazyFrame with schema evolution applied
 
     Example:
         >>> config = vf.Config(
         ...     alpha_dir=Path("/data/jyao/alpha"),
         ...     alpha_pattern="alpha_{date}.feather",
-        ...     alpha_preset="jyao_v20251114",
+        ...     alpha_schema="jyao_v20251114",
         ... )
         >>> vf.set_config(config)
         >>> df = vf.scan_alpha("20251114")
     """
     config = config or get_config()
     path = config.get_alpha_path(date)
-    df = _scan_file(path)
-    return _apply_alpha_mapping(df, config)
+    schema = _resolve_schema(config.alpha_schema)
+
+    df = _scan_file(path, schema=schema)
+    if schema:
+        df = _apply_schema_evolution(df, schema)
+
+    return df
 
 
 def scan_alphas(config: Config | None = None) -> pl.LazyFrame:
-    """Scan all alpha files with column mapping.
+    """Scan all alpha files with schema evolution.
 
     Args:
         config: Config to use, or get_config() if None
 
     Returns:
-        LazyFrame with column mapping and schema evolution applied
+        LazyFrame with schema evolution applied
 
     Raises:
         ValueError: If alpha_dir is not set or no files found
@@ -275,22 +233,37 @@ def scan_alphas(config: Config | None = None) -> pl.LazyFrame:
     if not files:
         raise ValueError(f"No files found matching {pattern} in {config.alpha_dir}")
 
-    dfs = [_scan_file(f) for f in files]
+    schema = _resolve_schema(config.alpha_schema)
+
+    dfs = [_scan_file(f, schema=schema) for f in files]
     df = pl.concat(dfs)
-    return _apply_alpha_mapping(df, config)
 
+    if schema:
+        df = _apply_schema_evolution(df, schema)
+
+    return df
 
-def _get_rename_map(preset: str | None) -> dict[str, str]:
-    """Get rename map from preset name.
+
+def load_calendar(config: Config | None = None) -> pl.DataFrame:
+    """Load trading calendar.
 
     Args:
-        preset: Preset name (e.g., "ylin_v20251204", "jyao_v20251114") or None
+        config: Config to use, or get_config() if None
 
     Returns:
-        Dict mapping old column names to new names
-    """
-    if not preset:
-        return {}
-    from .presets import PRESETS
+        DataFrame with date, prev_date, next_date columns
+
+    Raises:
+        ValueError: If calendar_path is not set in config
 
-    return PRESETS.get(preset.lower(), {})
+    Example:
+        >>> config = vf.Config(
+        ...     calendar_path=Path("/data/calendar.parquet")
+        ... )
+        >>> vf.set_config(config)
+        >>> calendar = vf.load_calendar()
+    """
+    config = config or get_config()
+    if config.calendar_path is None:
+        raise ValueError("calendar_path is not set in Config")
+    return pl.read_parquet(config.calendar_path)

vizflow/ops.py

@@ -157,6 +157,7 @@ def forward_return(
     alpha_time_col: str = "elapsed_ticktime",
     price_col: str = "mid",
     symbol_col: str = "ukey",
+    tolerance_ms: int = 5000,
 ) -> pl.LazyFrame:
     """Merge alpha's future price to trade and calculate forward returns.
 
@@ -177,6 +178,7 @@ def forward_return(
         alpha_time_col: Time column in alpha df (default: "elapsed_ticktime")
         price_col: Column name for price in both dfs (default: "mid")
         symbol_col: Symbol column for grouping (default: "ukey")
+        tolerance_ms: Max time difference in ms for asof join (default: 5000)
 
     Returns:
         Trade LazyFrame with forward_* and y_* columns added
@@ -210,6 +212,9 @@ def forward_return(
             (pl.col(trade_time_col) + horizon_ms).alias("_forward_time")
         )
 
+        # Sort by join columns (required for asof join)
+        trade = trade.sort([symbol_col, "_forward_time"])
+
         # Asof join: find alpha price at forward_time
         joined = trade.join_asof(
             alpha_lookup.rename({alpha_time_col: "_alpha_time", price_col: "_forward_price"}),
@@ -217,13 +222,16 @@ def forward_return(
             right_on="_alpha_time",
             by=symbol_col,
             strategy="nearest",
-            tolerance=1000,  # 1 second tolerance
+            tolerance=tolerance_ms,
         )
 
-        # Add forward price and calculate return
+        # Add forward price and calculate return (guard against zero price)
         trade = joined.with_columns([
             pl.col("_forward_price").alias(forward_col),
-            ((pl.col("_forward_price") - pl.col(price_col)) / pl.col(price_col)).alias(return_col),
+            pl.when(pl.col(price_col) != 0)
+            .then((pl.col("_forward_price") - pl.col(price_col)) / pl.col(price_col))
+            .otherwise(pl.lit(None))
+            .alias(return_col),
         ]).drop(["_forward_time", "_alpha_time", "_forward_price"])
 
     return trade.lazy()

vizflow/schema_evolution.py

@@ -0,0 +1,394 @@
+"""Schema Evolution for VizFlow.
+
+This module defines how raw data evolves into standard format through:
+- Column renaming (raw names → standard names)
+- Parse-time type specification (for CSV parsing)
+- Post-load type casting (e.g., Float64 → Int64)
+- Null value handling
+- Column exclusion
+
+Example:
+    >>> schema = SchemaEvolution(
+    ...     columns={
+    ...         "fillQty": ColumnSpec(
+    ...             rename_to="order_filled_qty",
+    ...             parse_dtype=pl.Float64,  # Parse as float (catch decimals)
+    ...             cast_dtype=pl.Int64,     # Then cast to int
+    ...         ),
+    ...     },
+    ...     null_values=["", "NA"],
+    ...     drop=["#HFTORD"],
+    ... )
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import polars as pl
+
+
+@dataclass
+class ColumnSpec:
+    """Specification for a single column's parsing and transformation.
+
+    Attributes:
+        rename_to: Standard column name after rename. None keeps original name.
+        parse_dtype: Type to use when parsing CSV. None uses Polars inference.
+        cast_dtype: Final type after post-load casting. None keeps parse type.
+
+    Examples:
+        # Rename only (most common)
+        ColumnSpec(rename_to="ukey")
+
+        # Parse as Float64, cast to Int64 (handle decimal errors in qty)
+        ColumnSpec(rename_to="order_filled_qty",
+                   parse_dtype=pl.Float64,
+                   cast_dtype=pl.Int64)
+
+        # Parse as specific type, no cast (trusted integer)
+        ColumnSpec(rename_to="timestamp", parse_dtype=pl.Int64)
+    """
+
+    rename_to: str | None = None
+    parse_dtype: Any = None  # pl.DataType
+    cast_dtype: Any = None   # pl.DataType
+
+
+@dataclass
+class SchemaEvolution:
+    """Defines how raw data evolves into standard format.
+
+    Combines column renaming, parse-time types, post-load casting,
+    null value handling, and column exclusion into a single structure.
+
+    Attributes:
+        columns: Mapping from original column name to ColumnSpec.
+        null_values: Strings to treat as null at parse time.
+        drop: Column names to exclude from output.
+        parent: Optional parent schema for version inheritance.
+
+    Example:
+        >>> YLIN_V20251204 = SchemaEvolution(
+        ...     columns={
+        ...         "symbol": ColumnSpec(rename_to="ukey", parse_dtype=pl.Int64),
+        ...         "fillQty": ColumnSpec(
+        ...             rename_to="order_filled_qty",
+        ...             parse_dtype=pl.Float64,
+        ...             cast_dtype=pl.Int64,
+        ...         ),
+        ...     },
+        ...     null_values=["", "NA", "null"],
+        ...     drop=["#HFTORD"],
+        ... )
+    """
+
+    columns: dict[str, ColumnSpec] = field(default_factory=dict)
+    null_values: list[str] = field(default_factory=lambda: ["", "NA", "null"])
+    drop: list[str] = field(default_factory=list)
+    parent: SchemaEvolution | None = None
+
+    def get_schema_overrides(self) -> dict[str, Any]:
+        """Get schema_overrides dict for pl.scan_csv().
+
+        Returns:
+            Mapping from original column name to Polars dtype.
+        """
+        result = {}
+        if self.parent:
+            result.update(self.parent.get_schema_overrides())
+        for col_name, spec in self.columns.items():
+            if spec.parse_dtype is not None:
+                result[col_name] = spec.parse_dtype
+        return result
+
+    def get_rename_map(self) -> dict[str, str]:
+        """Get rename mapping dict for df.rename().
+
+        Returns:
+            Mapping from original column name to new name.
+        """
+        result = {}
+        if self.parent:
+            result.update(self.parent.get_rename_map())
+        for col_name, spec in self.columns.items():
+            if spec.rename_to is not None:
+                result[col_name] = spec.rename_to
+        return result
+
+    def get_cast_map(self) -> dict[str, Any]:
+        """Get post-load cast mapping dict.
+
+        Returns:
+            Mapping from FINAL column name (after rename) to cast dtype.
+        """
+        result = {}
+        if self.parent:
+            result.update(self.parent.get_cast_map())
+        for col_name, spec in self.columns.items():
+            if spec.cast_dtype is not None:
+                final_name = spec.rename_to or col_name
+                result[final_name] = spec.cast_dtype
+        return result
+
+    def get_drop_columns(self) -> set[str]:
+        """Get set of columns to drop.
+
+        Returns:
+            Set of original column names to exclude.
+        """
+        result = set()
+        if self.parent:
+            result.update(self.parent.get_drop_columns())
+        result.update(self.drop)
+        return result
+
+    def get_null_values(self) -> list[str]:
+        """Get list of null value strings.
+
+        Returns:
+            List of strings to treat as null at parse time.
+        """
+        return self.null_values
+
+    def validate(self) -> list[str]:
+        """Validate schema configuration.
+
+        Returns:
+            List of warnings about potential issues.
+        """
+        warnings = []
+        for col_name, spec in self.columns.items():
+            if spec.cast_dtype is not None and spec.parse_dtype is None:
+                warnings.append(
+                    f"{col_name}: cast_dtype without parse_dtype may fail "
+                    "if Polars infers wrong type"
+                )
+        return warnings
+
+
+# =============================================================================
+# YLIN Trade Format (v2025-12-04)
+# =============================================================================
+
+YLIN_V20251204 = SchemaEvolution(
+    columns={
+        # === Order columns (18) ===
+        "symbol": ColumnSpec(rename_to="ukey", parse_dtype=pl.Int64),
+        "orderId": ColumnSpec(rename_to="order_id", parse_dtype=pl.Int64),
+        "orderSide": ColumnSpec(rename_to="order_side", parse_dtype=pl.String),
+        "orderQty": ColumnSpec(
+            rename_to="order_qty",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "orderPrice": ColumnSpec(rename_to="order_price", parse_dtype=pl.Float64),
+        "priceType": ColumnSpec(rename_to="order_price_type", parse_dtype=pl.String),
+        "fillQty": ColumnSpec(
+            rename_to="order_filled_qty",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "fillPrice": ColumnSpec(rename_to="fill_price", parse_dtype=pl.Float64),
+        "lastExchangeTs": ColumnSpec(rename_to="update_exchange_ts", parse_dtype=pl.Int64),
+        "createdTs": ColumnSpec(rename_to="create_exchange_ts", parse_dtype=pl.Int64),
+        "localTs": ColumnSpec(rename_to="create_local_ts", parse_dtype=pl.Int64),
+        "qtyAhead": ColumnSpec(
+            rename_to="qty_ahead",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "qtyBehind": ColumnSpec(
+            rename_to="qty_behind",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "orderStatus": ColumnSpec(rename_to="order_curr_state", parse_dtype=pl.String),
+        "orderTposType": ColumnSpec(rename_to="order_tpos_type", parse_dtype=pl.String),
+        "alphaTs": ColumnSpec(rename_to="alpha_ts", parse_dtype=pl.Int64),
+        "event": ColumnSpec(rename_to="event_type", parse_dtype=pl.String),
+        "cumFilledNotional": ColumnSpec(
+            rename_to="order_filled_notional",
+            parse_dtype=pl.Float64,
+        ),
+        # === Quote columns (20) ===
+        "bid": ColumnSpec(rename_to="bid_px0", parse_dtype=pl.Float64),
+        "bid2": ColumnSpec(rename_to="bid_px1", parse_dtype=pl.Float64),
+        "bid3": ColumnSpec(rename_to="bid_px2", parse_dtype=pl.Float64),
+        "bid4": ColumnSpec(rename_to="bid_px3", parse_dtype=pl.Float64),
+        "bid5": ColumnSpec(rename_to="bid_px4", parse_dtype=pl.Float64),
+        "ask": ColumnSpec(rename_to="ask_px0", parse_dtype=pl.Float64),
+        "ask2": ColumnSpec(rename_to="ask_px1", parse_dtype=pl.Float64),
+        "ask3": ColumnSpec(rename_to="ask_px2", parse_dtype=pl.Float64),
+        "ask4": ColumnSpec(rename_to="ask_px3", parse_dtype=pl.Float64),
+        "ask5": ColumnSpec(rename_to="ask_px4", parse_dtype=pl.Float64),
+        "bsize": ColumnSpec(
+            rename_to="bid_size0",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "bsize2": ColumnSpec(
+            rename_to="bid_size1",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "bsize3": ColumnSpec(
+            rename_to="bid_size2",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "bsize4": ColumnSpec(
+            rename_to="bid_size3",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "bsize5": ColumnSpec(
+            rename_to="bid_size4",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize": ColumnSpec(
+            rename_to="ask_size0",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize2": ColumnSpec(
+            rename_to="ask_size1",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize3": ColumnSpec(
+            rename_to="ask_size2",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize4": ColumnSpec(
+            rename_to="ask_size3",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize5": ColumnSpec(
+            rename_to="ask_size4",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "isRebasedQuote": ColumnSpec(rename_to="is_rebased", parse_dtype=pl.String),
+        "quoteSeqNum": ColumnSpec(rename_to="seq_num", parse_dtype=pl.Int64),
+        "quoteTs": ColumnSpec(rename_to="timestamp", parse_dtype=pl.Int64),
+        # === Position columns (11) ===
+        "startPos": ColumnSpec(
+            rename_to="init_net_pos",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "pos": ColumnSpec(
+            rename_to="current_net_pos",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "realizedPos": ColumnSpec(
+            rename_to="current_realized_net_pos",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "openBuyPos": ColumnSpec(
+            rename_to="open_buy",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "openSellPos": ColumnSpec(
+            rename_to="open_sell",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "cumBuy": ColumnSpec(
+            rename_to="cum_buy",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "cumSell": ColumnSpec(
+            rename_to="cum_sell",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "cashFlow": ColumnSpec(rename_to="cash_flow", parse_dtype=pl.Float64),
+        "frozenCash": ColumnSpec(rename_to="frozen_cash", parse_dtype=pl.Float64),
+        "globalCumBuyNotional": ColumnSpec(
+            rename_to="cum_buy_filled_notional",
+            parse_dtype=pl.Float64,
+        ),
+        "globalCumSellNotional": ColumnSpec(
+            rename_to="cum_sell_filled_notional",
+            parse_dtype=pl.Float64,
+        ),
+    },
+    null_values=["", "NA", "null", "NULL"],
+    drop=["#HFTORD"],
+)
+
+
+# =============================================================================
+# JYAO Alpha Format (v2025-11-14)
+# =============================================================================
+
+JYAO_V20251114 = SchemaEvolution(
+    columns={
+        # Symbol column
+        "ukey": ColumnSpec(parse_dtype=pl.Int64),  # No rename, just parse type
+        # Quote columns
+        "BidPrice1": ColumnSpec(rename_to="bid_px0", parse_dtype=pl.Float64),
+        "AskPrice1": ColumnSpec(rename_to="ask_px0", parse_dtype=pl.Float64),
+        "BidVolume1": ColumnSpec(
+            rename_to="bid_size0",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "AskVolume1": ColumnSpec(
+            rename_to="ask_size0",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        # Time columns
+        "TimeStamp": ColumnSpec(rename_to="timestamp", parse_dtype=pl.Int64),
+        "GlobalExTime": ColumnSpec(rename_to="global_exchange_ts", parse_dtype=pl.Int64),
+        "DataDate": ColumnSpec(rename_to="data_date", parse_dtype=pl.String),
+        # Volume
+        "Volume": ColumnSpec(
+            rename_to="volume",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        # Predictor columns (x_* = alpha predictions)
+        # Rule: ≤60s → s, >60s → m
+        "x10s": ColumnSpec(rename_to="x_10s", parse_dtype=pl.Float64),
+        "x60s": ColumnSpec(rename_to="x_60s", parse_dtype=pl.Float64),
+        "alpha1": ColumnSpec(rename_to="x_3m", parse_dtype=pl.Float64),
+        "alpha2": ColumnSpec(rename_to="x_30m", parse_dtype=pl.Float64),
+    },
+    null_values=["", "NA"],
+)
+
+
+# =============================================================================
+# Schema Registry
+# =============================================================================
+
+SCHEMAS: dict[str, SchemaEvolution] = {
+    "ylin_v20251204": YLIN_V20251204,
+    "jyao_v20251114": JYAO_V20251114,
+}
+
+
+def get_schema(name: str | None) -> SchemaEvolution | None:
+    """Get SchemaEvolution by name.
+
+    Args:
+        name: Schema name (e.g., "ylin_v20251204") or None.
+
+    Returns:
+        SchemaEvolution or None if name is None or not found.
+    """
+    if not name:
+        return None
+    return SCHEMAS.get(name.lower())

{vizflow-0.5.0.dist-info → vizflow-0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vizflow
-Version: 0.5.0
+Version: 0.5.2
 Requires-Python: >=3.9
 Requires-Dist: polars>=0.20.0
 Provides-Extra: dev

vizflow-0.5.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+vizflow/__init__.py,sha256=nmZ9_4DkT6ndFefemioNGNw9ELWFCQsQASxBxLHoAZs,529
+vizflow/config.py,sha256=y4vRvjVTa1H5AdQf0q_XhYr-3EBDJst8BJq52ODN3uk,6456
+vizflow/io.py,sha256=eheqyLHGiSh69erxMk98FK-GYycbSheqkrIYRYGFy3A,7687
+vizflow/market.py,sha256=MtNz_nnZxC66Aq-i2PXEwaFCTknijFWYZUUv6798k2s,2493
+vizflow/ops.py,sha256=4UwxOTPhvZ1_4PI3pcxbXfLAYsn1Ecj6nyBtBBr7KS8,7761
+vizflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vizflow/schema_evolution.py,sha256=7ZgybN7aS6SWxteTnX_uXZWn-IfosIPzr42_f3BlFv8,13909
+vizflow-0.5.2.dist-info/METADATA,sha256=DQdMKOm4yHQ4REnaOptw5avVjlaCpuXbQLFwHbH9_Gk,388
+vizflow-0.5.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+vizflow-0.5.2.dist-info/RECORD,,

vizflow/presets.py

@@ -1,87 +0,0 @@
-"""Column mapping presets for VizFlow."""
-
-# ylin's trade format (v2025-12-04)
-YLIN_V20251204 = {
-    # Order columns (18)
-    "symbol": "ukey",
-    "orderId": "order_id",
-    "orderSide": "order_side",
-    "orderQty": "order_qty",
-    "orderPrice": "order_price",
-    "priceType": "order_price_type",
-    "fillQty": "order_filled_qty",
-    "fillPrice": "fill_price",
-    "lastExchangeTs": "update_exchange_ts",
-    "createdTs": "create_exchange_ts",
-    "localTs": "create_local_ts",
-    "qtyAhead": "qty_ahead",
-    "qtyBehind": "qty_behind",
-    "orderStatus": "order_curr_state",
-    "orderTposType": "order_tpos_type",
-    "alphaTs": "alpha_ts",
-    "event": "event_type",
-    "cumFilledNotional": "order_filled_notional",
-    # Quote columns (15)
-    "bid": "bid_px0",
-    "bid2": "bid_px1",
-    "bid3": "bid_px2",
-    "bid4": "bid_px3",
-    "bid5": "bid_px4",
-    "ask": "ask_px0",
-    "ask2": "ask_px1",
-    "ask3": "ask_px2",
-    "ask4": "ask_px3",
-    "ask5": "ask_px4",
-    "bsize": "bid_size0",
-    "bsize2": "bid_size1",
-    "bsize3": "bid_size2",
-    "bsize4": "bid_size3",
-    "bsize5": "bid_size4",
-    "asize": "ask_size0",
-    "asize2": "ask_size1",
-    "asize3": "ask_size2",
-    "asize4": "ask_size3",
-    "asize5": "ask_size4",
-    "isRebasedQuote": "is_rebased",
-    "quoteSeqNum": "seq_num",
-    "quoteTs": "timestamp",
-    # Position columns (11)
-    "startPos": "init_net_pos",
-    "pos": "current_net_pos",
-    "realizedPos": "current_realized_net_pos",
-    "openBuyPos": "open_buy",
-    "openSellPos": "open_sell",
-    "cumBuy": "cum_buy",
-    "cumSell": "cum_sell",
-    "cashFlow": "cash_flow",
-    "frozenCash": "frozen_cash",
-    "globalCumBuyNotional": "cum_buy_filled_notional",
-    "globalCumSellNotional": "cum_sell_filled_notional",
-}
-
-# jyao's alpha format (v2025-11-14)
-JYAO_V20251114 = {
-    # Quote columns
-    "BidPrice1": "bid_px0",
-    "AskPrice1": "ask_px0",
-    "BidVolume1": "bid_size0",
-    "AskVolume1": "ask_size0",
-    # Time columns
-    "TimeStamp": "timestamp",
-    "GlobalExTime": "global_exchange_ts",
-    "DataDate": "data_date",
-    # Volume
-    "Volume": "volume",
-    # Predictor columns (x_* = alpha predictions)
-    # Rule: ≤60s → s, >60s → m
-    "x10s": "x_10s",
-    "x60s": "x_60s",
-    "alpha1": "x_3m",
-    "alpha2": "x_30m",
-}
-
-# Preset registry for dynamic lookup
-PRESETS: dict[str, dict[str, str]] = {
-    "ylin_v20251204": YLIN_V20251204,
-    "jyao_v20251114": JYAO_V20251114,
-}

vizflow-0.5.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-vizflow/__init__.py,sha256=ZIMYQ-Yzh3eEAd7MSqNA00SlSuj45bEE6NsXM9Qc6O0,496
-vizflow/config.py,sha256=zSZnhdHzgXSqhDenHcHKm4CDGrMpKAdkNNRoUYYF1uc,6530
-vizflow/io.py,sha256=zmN0fFQOTmSBEBKangMExr0Q5mC2gajZM6GgdAyWkw4,8824
-vizflow/market.py,sha256=MtNz_nnZxC66Aq-i2PXEwaFCTknijFWYZUUv6798k2s,2493
-vizflow/ops.py,sha256=6hKOjJowFrw1b6z4y8Liea9KTp8Fgy6kCGPZ6t15PVk,7426
-vizflow/presets.py,sha256=h91NZoOH4YAx0bbsaNigECf9WOcWh1QZavguunWkaLE,2452
-vizflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vizflow-0.5.0.dist-info/METADATA,sha256=cIIvBPZo2U6Sp46Wxgyu_tCVPWF4DGsgvapbavEBGl8,388
-vizflow-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-vizflow-0.5.0.dist-info/RECORD,,