vizflow 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

vizflow/__init__.py

@@ -5,13 +5,11 @@ Usage:
     import vizflow as vf
 """
 
-__version__ = "0.5.1"
+__version__ = "0.5.3"
 
-from .config import ColumnSchema, Config, get_config, set_config
+from .config import Config, get_config, set_config
 from .io import (
-    load_alpha,
     load_calendar,
-    load_trade,
     scan_alpha,
     scan_alphas,
     scan_trade,
@@ -19,4 +17,11 @@ from .io import (
 )
 from .market import CN, CRYPTO, Market, Session
 from .ops import aggregate, bin, forward_return, parse_time
-from .presets import JYAO_V20251114, PRESETS, YLIN_V20251204
+from .schema_evolution import (
+    JYAO_V20251114,
+    SCHEMAS,
+    YLIN_V20251204,
+    ColumnSpec,
+    SchemaEvolution,
+    get_schema,
+)

vizflow/config.py

@@ -4,7 +4,10 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .schema_evolution import SchemaEvolution
 
 # Global config instance
 _global_config: Config | None = None
@@ -25,21 +28,6 @@ def _validate_date(date: str) -> None:
         )
 
 
-@dataclass
-class ColumnSchema:
-    """Schema for a column with type casting.
-
-    Attributes:
-        cast_to: Target type after casting (e.g. pl.Int64)
-
-    Example:
-        # Handle float precision errors: 1.00000002 → 1
-        ColumnSchema(cast_to=pl.Int64)
-    """
-
-    cast_to: Any  # pl.DataType, but avoid import for now
-
-
 @dataclass
 class Config:
     """Central configuration for a pipeline run.
@@ -53,14 +41,20 @@ class Config:
         replay_dir: Directory for FIFO replay output (materialization 1)
         aggregate_dir: Directory for aggregation output (materialization 2)
         market: Market identifier, e.g. "CN"
-        alpha_columns: Mapping from semantic names to alpha column names
-        trade_columns: Mapping from semantic names to trade column names
-        alpha_schema: Schema evolution for alpha columns
-        trade_schema: Schema evolution for trade columns
+        trade_schema: Schema evolution for trade data (name or SchemaEvolution)
+        alpha_schema: Schema evolution for alpha data (name or SchemaEvolution)
         binwidths: Mapping from column names to bin widths
         group_by: Columns to group by in aggregation
         horizons: List of forward return horizons in seconds
         time_cutoff: Optional time cutoff (e.g. 143000000 for 14:30:00)
+
+    Example:
+        >>> config = vf.Config(
+        ...     trade_dir=Path("data/ylin/trade"),
+        ...     trade_pattern="{date}.meords",
+        ...     trade_schema="ylin_v20251204",  # Use registered schema by name
+        ...     market="CN",
+        ... )
     """
 
     # === Input Paths ===
@@ -77,17 +71,10 @@ class Config:
     # === Market ===
     market: str = "CN"
 
-    # === Column Mapping ===
-    alpha_columns: dict[str, str] = field(default_factory=dict)
-    trade_columns: dict[str, str] = field(default_factory=dict)
-
     # === Schema Evolution ===
-    alpha_schema: dict[str, ColumnSchema] = field(default_factory=dict)
-    trade_schema: dict[str, ColumnSchema] = field(default_factory=dict)
-
-    # === Column Mapping Presets ===
-    trade_preset: str | None = None  # "ylin" or None
-    alpha_preset: str | None = None  # "jyao_v20251114" or None
+    # Can be a string (schema name) or SchemaEvolution instance
+    trade_schema: str | SchemaEvolution | None = None
+    alpha_schema: str | SchemaEvolution | None = None
 
     # === Aggregation ===
     binwidths: dict[str, float] = field(default_factory=dict)
@@ -114,20 +101,6 @@ class Config:
         if isinstance(self.aggregate_dir, str):
             self.aggregate_dir = Path(self.aggregate_dir)
 
-    def col(self, semantic: str, source: str = "trade") -> str:
-        """Get actual column name from semantic name.
-
-        Args:
-            semantic: Semantic column name (e.g. "timestamp", "price")
-            source: "alpha" or "trade"
-
-        Returns:
-            Actual column name, or the semantic name if no mapping exists
-        """
-        if source == "alpha":
-            return self.alpha_columns.get(semantic, semantic)
-        return self.trade_columns.get(semantic, semantic)
-
     def get_alpha_path(self, date: str) -> Path:
         """Get alpha file path for a date.
 

vizflow/io.py

@@ -4,108 +4,40 @@ from __future__ import annotations
 
 import polars as pl
 
-from .config import ColumnSchema, Config, get_config
+from .config import Config, get_config
+from .schema_evolution import SchemaEvolution, get_schema
 
 
-def _apply_schema(df: pl.LazyFrame, schema: dict[str, ColumnSchema]) -> pl.LazyFrame:
-    """Apply type casting from schema definition.
+def _resolve_schema(
+    schema_ref: str | SchemaEvolution | None,
+) -> SchemaEvolution | None:
+    """Resolve schema reference to SchemaEvolution instance.
 
     Args:
-        df: LazyFrame to apply schema to
-        schema: Mapping from column names to ColumnSchema
+        schema_ref: Schema name string, SchemaEvolution instance, or None.
 
     Returns:
-        LazyFrame with type casts applied
+        SchemaEvolution instance or None.
     """
-    for col_name, col_schema in schema.items():
-        df = df.with_columns(pl.col(col_name).cast(col_schema.cast_to))
-    return df
-
-
-def load_alpha(date: str, config: Config | None = None) -> pl.LazyFrame:
-    """Load alpha data for a date with automatic schema evolution.
-
-    Args:
-        date: Date string, e.g. "20241001"
-        config: Config to use, or get_config() if None
-
-    Returns:
-        LazyFrame with schema evolution applied
-
-    Example:
-        >>> config = vf.Config(
-        ...     output_dir=Path("/data/output"),
-        ...     alpha_dir=Path("/data/alpha"),
-        ...     alpha_schema={"qty": vf.ColumnSchema(cast_to=pl.Int64)}
-        ... )
-        >>> vf.set_config(config)
-        >>> alpha = vf.load_alpha("20241001")
-    """
-    config = config or get_config()
-    path = config.get_alpha_path(date)
-    df = pl.scan_ipc(path)
-    return _apply_schema(df, config.alpha_schema)
+    if schema_ref is None:
+        return None
+    if isinstance(schema_ref, SchemaEvolution):
+        return schema_ref
+    return get_schema(schema_ref)
 
 
-def load_trade(date: str, config: Config | None = None) -> pl.LazyFrame:
-    """Load trade data for a date with automatic schema evolution.
+def _scan_file(
+    path,
+    schema: SchemaEvolution | None = None,
+) -> pl.LazyFrame:
+    """Scan a file based on its extension with optional schema.
 
     Args:
-        date: Date string, e.g. "20241001"
-        config: Config to use, or get_config() if None
+        path: Path to file.
+        schema: Optional SchemaEvolution for CSV parsing options.
 
     Returns:
-        LazyFrame with schema evolution applied
-
-    Example:
-        >>> config = vf.Config(
-        ...     output_dir=Path("/data/output"),
-        ...     trade_dir=Path("/data/trade"),
-        ...     trade_schema={"qty": vf.ColumnSchema(cast_to=pl.Int64)}
-        ... )
-        >>> vf.set_config(config)
-        >>> trade = vf.load_trade("20241001")  # qty: 1.00000002 → 1
-    """
-    config = config or get_config()
-    path = config.get_trade_path(date)
-    df = pl.scan_ipc(path)
-    return _apply_schema(df, config.trade_schema)
-
-
-def load_calendar(config: Config | None = None) -> pl.DataFrame:
-    """Load trading calendar.
-
-    Args:
-        config: Config to use, or get_config() if None
-
-    Returns:
-        DataFrame with date, prev_date, next_date columns
-
-    Raises:
-        ValueError: If calendar_path is not set in config
-
-    Example:
-        >>> config = vf.Config(
-        ...     output_dir=Path("/data/output"),
-        ...     calendar_path=Path("/data/calendar.parquet")
-        ... )
-        >>> vf.set_config(config)
-        >>> calendar = vf.load_calendar()
-    """
-    config = config or get_config()
-    if config.calendar_path is None:
-        raise ValueError("calendar_path is not set in Config")
-    return pl.read_parquet(config.calendar_path)
-
-
-def _scan_file(path) -> pl.LazyFrame:
-    """Scan a file based on its extension.
-
-    Args:
-        path: Path to file
-
-    Returns:
-        LazyFrame from the file
+        LazyFrame from the file.
 
     Supported formats:
         - .feather, .ipc, .arrow: IPC format (pl.scan_ipc)
@@ -117,7 +49,15 @@ def _scan_file(path) -> pl.LazyFrame:
     if suffix in ("feather", "ipc", "arrow"):
         return pl.scan_ipc(path)
     elif suffix in ("csv", "meords"):
-        return pl.scan_csv(path)
+        csv_kwargs = {}
+        if schema:
+            schema_overrides = schema.get_schema_overrides()
+            if schema_overrides:
+                csv_kwargs["schema_overrides"] = schema_overrides
+            null_values = schema.get_null_values()
+            if null_values:
+                csv_kwargs["null_values"] = null_values
+        return pl.scan_csv(path, **csv_kwargs)
     elif suffix == "parquet":
         return pl.scan_parquet(path)
     else:
@@ -127,50 +67,96 @@ def _scan_file(path) -> pl.LazyFrame:
         )
 
 
+def _apply_schema_evolution(
+    df: pl.LazyFrame,
+    schema: SchemaEvolution,
+) -> pl.LazyFrame:
+    """Apply full schema evolution: drop, rename, cast.
+
+    Args:
+        df: LazyFrame to transform.
+        schema: SchemaEvolution with transformation rules.
+
+    Returns:
+        Transformed LazyFrame.
+    """
+    existing = set(df.collect_schema().names())
+
+    # Step 1: Drop excluded columns
+    drop_cols = schema.get_drop_columns()
+    to_drop = [c for c in drop_cols if c in existing]
+    if to_drop:
+        df = df.drop(to_drop)
+        existing -= set(to_drop)
+
+    # Step 2: Rename columns
+    rename_map = schema.get_rename_map()
+    to_rename = {k: v for k, v in rename_map.items() if k in existing}
+    if to_rename:
+        df = df.rename(to_rename)
+        # Update existing names after rename
+        for old, new in to_rename.items():
+            existing.discard(old)
+            existing.add(new)
+
+    # Step 3: Cast columns (using FINAL names after rename)
+    cast_map = schema.get_cast_map()
+    for col_name, dtype in cast_map.items():
+        if col_name in existing:
+            df = df.with_columns(pl.col(col_name).cast(dtype))
+
+    return df
+
+
 def scan_trade(date: str, config: Config | None = None) -> pl.LazyFrame:
-    """Scan single date trade file with column mapping.
+    """Scan single date trade file with schema evolution.
 
-    Supports both IPC/feather format and CSV format (including .meords files).
+    Supports IPC/feather, CSV (including .meords), and Parquet formats.
 
     Args:
         date: Date string, e.g. "20241001"
         config: Config to use, or get_config() if None
 
     Returns:
-        LazyFrame with column mapping and schema evolution applied
+        LazyFrame with schema evolution applied
 
     Example:
         >>> config = vf.Config(
-        ...     trade_dir=Path("/data/yuanzhao/"),
+        ...     trade_dir=Path("/data/ylin/trade"),
         ...     trade_pattern="{date}.meords",
-        ...     trade_preset="ylin_v20251204",
+        ...     trade_schema="ylin_v20251204",
         ... )
         >>> vf.set_config(config)
         >>> df = vf.scan_trade("20241001")
     """
     config = config or get_config()
     path = config.get_trade_path(date)
-    df = _scan_file(path)
-    return _apply_trade_mapping(df, config)
+    schema = _resolve_schema(config.trade_schema)
+
+    df = _scan_file(path, schema=schema)
+    if schema:
+        df = _apply_schema_evolution(df, schema)
+
+    return df
 
 
 def scan_trades(config: Config | None = None) -> pl.LazyFrame:
-    """Scan all trade files with column mapping.
+    """Scan all trade files with schema evolution.
 
     Args:
         config: Config to use, or get_config() if None
 
     Returns:
-        LazyFrame with column mapping and schema evolution applied
+        LazyFrame with schema evolution applied
 
     Raises:
         ValueError: If trade_dir is not set or no files found
 
     Example:
         >>> config = vf.Config(
-        ...     trade_dir=Path("/data/yuanzhao/"),
-        ...     trade_pattern="{date}.feather",
-        ...     trade_preset="ylin_v20251204",
+        ...     trade_dir=Path("/data/ylin/trade"),
+        ...     trade_pattern="{date}.meords",
+        ...     trade_schema="ylin_v20251204",
         ... )
         >>> vf.set_config(config)
         >>> df = vf.scan_trades()
@@ -184,82 +170,56 @@ def scan_trades(config: Config | None = None) -> pl.LazyFrame:
     if not files:
         raise ValueError(f"No files found matching {pattern} in {config.trade_dir}")
 
+    schema = _resolve_schema(config.trade_schema)
+
     # Concatenate all files using lazy scanning
-    dfs = [_scan_file(f) for f in files]
+    dfs = [_scan_file(f, schema=schema) for f in files]
     df = pl.concat(dfs)
-    return _apply_trade_mapping(df, config)
-
-
-def _apply_trade_mapping(df: pl.LazyFrame, config: Config) -> pl.LazyFrame:
-    """Apply column rename + schema evolution for trade data."""
-    df = _apply_rename(df, config.trade_preset)
-    return _apply_schema(df, config.trade_schema)
-
-
-def _apply_alpha_mapping(df: pl.LazyFrame, config: Config) -> pl.LazyFrame:
-    """Apply column rename + schema evolution for alpha data."""
-    df = _apply_rename(df, config.alpha_preset)
-    return _apply_schema(df, config.alpha_schema)
-
-
-def _apply_rename(df: pl.LazyFrame, preset: str | None) -> pl.LazyFrame:
-    """Apply column rename from preset name.
-
-    Args:
-        df: LazyFrame to rename columns
-        preset: Preset name (e.g., "ylin", "jyao_v20251114") or None
-    """
-    # Drop record type prefix column if present (from CSV files)
-    existing = set(df.collect_schema().names())
-    if "#HFTORD" in existing:
-        df = df.drop("#HFTORD")
-        existing.remove("#HFTORD")
 
-    # Get rename map from preset
-    rename_map = _get_rename_map(preset)
-
-    if rename_map:
-        existing = set(df.collect_schema().names())
-        to_rename = {k: v for k, v in rename_map.items() if k in existing}
-        if to_rename:
-            df = df.rename(to_rename)
+    if schema:
+        df = _apply_schema_evolution(df, schema)
 
     return df
 
 
 def scan_alpha(date: str, config: Config | None = None) -> pl.LazyFrame:
-    """Scan single date alpha file with column mapping.
+    """Scan single date alpha file with schema evolution.
 
     Args:
         date: Date string, e.g. "20241001"
         config: Config to use, or get_config() if None
 
     Returns:
-        LazyFrame with column mapping and schema evolution applied
+        LazyFrame with schema evolution applied
 
     Example:
         >>> config = vf.Config(
         ...     alpha_dir=Path("/data/jyao/alpha"),
         ...     alpha_pattern="alpha_{date}.feather",
-        ...     alpha_preset="jyao_v20251114",
+        ...     alpha_schema="jyao_v20251114",
         ... )
         >>> vf.set_config(config)
         >>> df = vf.scan_alpha("20251114")
     """
     config = config or get_config()
     path = config.get_alpha_path(date)
-    df = _scan_file(path)
-    return _apply_alpha_mapping(df, config)
+    schema = _resolve_schema(config.alpha_schema)
+
+    df = _scan_file(path, schema=schema)
+    if schema:
+        df = _apply_schema_evolution(df, schema)
+
+    return df
 
 
 def scan_alphas(config: Config | None = None) -> pl.LazyFrame:
-    """Scan all alpha files with column mapping.
+    """Scan all alpha files with schema evolution.
 
     Args:
         config: Config to use, or get_config() if None
 
     Returns:
-        LazyFrame with column mapping and schema evolution applied
+        LazyFrame with schema evolution applied
 
     Raises:
         ValueError: If alpha_dir is not set or no files found
@@ -273,22 +233,37 @@ def scan_alphas(config: Config | None = None) -> pl.LazyFrame:
     if not files:
         raise ValueError(f"No files found matching {pattern} in {config.alpha_dir}")
 
-    dfs = [_scan_file(f) for f in files]
+    schema = _resolve_schema(config.alpha_schema)
+
+    dfs = [_scan_file(f, schema=schema) for f in files]
     df = pl.concat(dfs)
-    return _apply_alpha_mapping(df, config)
 
+    if schema:
+        df = _apply_schema_evolution(df, schema)
 
-def _get_rename_map(preset: str | None) -> dict[str, str]:
-    """Get rename map from preset name.
+    return df
+
+
+def load_calendar(config: Config | None = None) -> pl.DataFrame:
+    """Load trading calendar.
 
     Args:
-        preset: Preset name (e.g., "ylin_v20251204", "jyao_v20251114") or None
+        config: Config to use, or get_config() if None
 
     Returns:
-        Dict mapping old column names to new names
-    """
-    if not preset:
-        return {}
-    from .presets import PRESETS
+        DataFrame with date, prev_date, next_date columns
 
-    return PRESETS.get(preset.lower(), {})
+    Raises:
+        ValueError: If calendar_path is not set in config
+
+    Example:
+        >>> config = vf.Config(
+        ...     calendar_path=Path("/data/calendar.parquet")
+        ... )
+        >>> vf.set_config(config)
+        >>> calendar = vf.load_calendar()
+    """
+    config = config or get_config()
+    if config.calendar_path is None:
+        raise ValueError("calendar_path is not set in Config")
+    return pl.read_parquet(config.calendar_path)

vizflow/schema_evolution.py

@@ -0,0 +1,395 @@
+"""Schema Evolution for VizFlow.
+
+This module defines how raw data evolves into standard format through:
+- Column renaming (raw names → standard names)
+- Parse-time type specification (for CSV parsing)
+- Post-load type casting (e.g., Float64 → Int64)
+- Null value handling
+- Column exclusion
+
+Example:
+    >>> schema = SchemaEvolution(
+    ...     columns={
+    ...         "fillQty": ColumnSpec(
+    ...             rename_to="order_filled_qty",
+    ...             parse_dtype=pl.Float64,  # Parse as float (catch decimals)
+    ...             cast_dtype=pl.Int64,     # Then cast to int
+    ...         ),
+    ...     },
+    ...     null_values=["", "NA"],
+    ...     drop=["#HFTORD"],
+    ... )
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import polars as pl
+
+
+@dataclass
+class ColumnSpec:
+    """Specification for a single column's parsing and transformation.
+
+    Attributes:
+        rename_to: Standard column name after rename. None keeps original name.
+        parse_dtype: Type to use when parsing CSV. None uses Polars inference.
+        cast_dtype: Final type after post-load casting. None keeps parse type.
+
+    Examples:
+        # Rename only (most common)
+        ColumnSpec(rename_to="ukey")
+
+        # Parse as Float64, cast to Int64 (handle decimal errors in qty)
+        ColumnSpec(rename_to="order_filled_qty",
+                   parse_dtype=pl.Float64,
+                   cast_dtype=pl.Int64)
+
+        # Parse as specific type, no cast (trusted integer)
+        ColumnSpec(rename_to="timestamp", parse_dtype=pl.Int64)
+    """
+
+    rename_to: str | None = None
+    parse_dtype: Any = None  # pl.DataType
+    cast_dtype: Any = None   # pl.DataType
+
+
+@dataclass
+class SchemaEvolution:
+    """Defines how raw data evolves into standard format.
+
+    Combines column renaming, parse-time types, post-load casting,
+    null value handling, and column exclusion into a single structure.
+
+    Attributes:
+        columns: Mapping from original column name to ColumnSpec.
+        null_values: Strings to treat as null at parse time.
+        drop: Column names to exclude from output.
+        parent: Optional parent schema for version inheritance.
+
+    Example:
+        >>> YLIN_V20251204 = SchemaEvolution(
+        ...     columns={
+        ...         "symbol": ColumnSpec(rename_to="ukey", parse_dtype=pl.Int64),
+        ...         "fillQty": ColumnSpec(
+        ...             rename_to="order_filled_qty",
+        ...             parse_dtype=pl.Float64,
+        ...             cast_dtype=pl.Int64,
+        ...         ),
+        ...     },
+        ...     null_values=["", "NA", "null"],
+        ...     drop=["#HFTORD"],
+        ... )
+    """
+
+    columns: dict[str, ColumnSpec] = field(default_factory=dict)
+    null_values: list[str] = field(default_factory=lambda: ["", "NA", "null"])
+    drop: list[str] = field(default_factory=list)
+    parent: SchemaEvolution | None = None
+
+    def get_schema_overrides(self) -> dict[str, Any]:
+        """Get schema_overrides dict for pl.scan_csv().
+
+        Returns:
+            Mapping from original column name to Polars dtype.
+        """
+        result = {}
+        if self.parent:
+            result.update(self.parent.get_schema_overrides())
+        for col_name, spec in self.columns.items():
+            if spec.parse_dtype is not None:
+                result[col_name] = spec.parse_dtype
+        return result
+
+    def get_rename_map(self) -> dict[str, str]:
+        """Get rename mapping dict for df.rename().
+
+        Returns:
+            Mapping from original column name to new name.
+        """
+        result = {}
+        if self.parent:
+            result.update(self.parent.get_rename_map())
+        for col_name, spec in self.columns.items():
+            if spec.rename_to is not None:
+                result[col_name] = spec.rename_to
+        return result
+
+    def get_cast_map(self) -> dict[str, Any]:
+        """Get post-load cast mapping dict.
+
+        Returns:
+            Mapping from FINAL column name (after rename) to cast dtype.
+        """
+        result = {}
+        if self.parent:
+            result.update(self.parent.get_cast_map())
+        for col_name, spec in self.columns.items():
+            if spec.cast_dtype is not None:
+                final_name = spec.rename_to or col_name
+                result[final_name] = spec.cast_dtype
+        return result
+
+    def get_drop_columns(self) -> set[str]:
+        """Get set of columns to drop.
+
+        Returns:
+            Set of original column names to exclude.
+        """
+        result = set()
+        if self.parent:
+            result.update(self.parent.get_drop_columns())
+        result.update(self.drop)
+        return result
+
+    def get_null_values(self) -> list[str]:
+        """Get list of null value strings.
+
+        Returns:
+            List of strings to treat as null at parse time.
+        """
+        return self.null_values
+
+    def validate(self) -> list[str]:
+        """Validate schema configuration.
+
+        Returns:
+            List of warnings about potential issues.
+        """
+        warnings = []
+        for col_name, spec in self.columns.items():
+            if spec.cast_dtype is not None and spec.parse_dtype is None:
+                warnings.append(
+                    f"{col_name}: cast_dtype without parse_dtype may fail "
+                    "if Polars infers wrong type"
+                )
+        return warnings
+
+
+# =============================================================================
+# YLIN Trade Format (v2025-12-04)
+# =============================================================================
+
+YLIN_V20251204 = SchemaEvolution(
+    columns={
+        # === Order columns (18) ===
+        "symbol": ColumnSpec(rename_to="ukey", parse_dtype=pl.Int64),
+        "orderId": ColumnSpec(rename_to="order_id", parse_dtype=pl.Int64),
+        "orderSide": ColumnSpec(rename_to="order_side", parse_dtype=pl.String),
+        "orderQty": ColumnSpec(
+            rename_to="order_qty",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "orderPrice": ColumnSpec(rename_to="order_price", parse_dtype=pl.Float64),
+        "priceType": ColumnSpec(rename_to="order_price_type", parse_dtype=pl.String),
+        "fillQty": ColumnSpec(
+            rename_to="order_filled_qty",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "fillPrice": ColumnSpec(rename_to="fill_price", parse_dtype=pl.Float64),
+        "lastExchangeTs": ColumnSpec(rename_to="update_exchange_ts", parse_dtype=pl.Int64),
+        "createdTs": ColumnSpec(rename_to="create_exchange_ts", parse_dtype=pl.Int64),
+        "localTs": ColumnSpec(rename_to="create_local_ts", parse_dtype=pl.Int64),
+        "qtyAhead": ColumnSpec(
+            rename_to="qty_ahead",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "qtyBehind": ColumnSpec(
+            rename_to="qty_behind",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "orderStatus": ColumnSpec(rename_to="order_curr_state", parse_dtype=pl.String),
+        "orderTposType": ColumnSpec(rename_to="order_tpos_type", parse_dtype=pl.String),
+        "alphaTs": ColumnSpec(rename_to="alpha_ts", parse_dtype=pl.Int64),
+        "event": ColumnSpec(rename_to="event_type", parse_dtype=pl.String),
+        "cumFilledNotional": ColumnSpec(
+            rename_to="order_filled_notional",
+            parse_dtype=pl.Float64,
+        ),
+        # === Quote columns (20) ===
+        "bid": ColumnSpec(rename_to="bid_px0", parse_dtype=pl.Float64),
+        "bid2": ColumnSpec(rename_to="bid_px1", parse_dtype=pl.Float64),
+        "bid3": ColumnSpec(rename_to="bid_px2", parse_dtype=pl.Float64),
+        "bid4": ColumnSpec(rename_to="bid_px3", parse_dtype=pl.Float64),
+        "bid5": ColumnSpec(rename_to="bid_px4", parse_dtype=pl.Float64),
+        "ask": ColumnSpec(rename_to="ask_px0", parse_dtype=pl.Float64),
+        "ask2": ColumnSpec(rename_to="ask_px1", parse_dtype=pl.Float64),
+        "ask3": ColumnSpec(rename_to="ask_px2", parse_dtype=pl.Float64),
+        "ask4": ColumnSpec(rename_to="ask_px3", parse_dtype=pl.Float64),
+        "ask5": ColumnSpec(rename_to="ask_px4", parse_dtype=pl.Float64),
+        "bsize": ColumnSpec(
+            rename_to="bid_size0",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "bsize2": ColumnSpec(
+            rename_to="bid_size1",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "bsize3": ColumnSpec(
+            rename_to="bid_size2",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "bsize4": ColumnSpec(
+            rename_to="bid_size3",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "bsize5": ColumnSpec(
+            rename_to="bid_size4",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize": ColumnSpec(
+            rename_to="ask_size0",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize2": ColumnSpec(
+            rename_to="ask_size1",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize3": ColumnSpec(
+            rename_to="ask_size2",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize4": ColumnSpec(
+            rename_to="ask_size3",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "asize5": ColumnSpec(
+            rename_to="ask_size4",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "isRebasedQuote": ColumnSpec(rename_to="is_rebased", parse_dtype=pl.String),
+        "quoteSeqNum": ColumnSpec(rename_to="seq_num", parse_dtype=pl.Int64),
+        "quoteTs": ColumnSpec(rename_to="timestamp", parse_dtype=pl.Int64),
+        # === Position columns (11) ===
+        "startPos": ColumnSpec(
+            rename_to="init_net_pos",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "pos": ColumnSpec(
+            rename_to="current_net_pos",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "realizedPos": ColumnSpec(
+            rename_to="current_realized_net_pos",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "openBuyPos": ColumnSpec(
+            rename_to="open_buy",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "openSellPos": ColumnSpec(
+            rename_to="open_sell",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "cumBuy": ColumnSpec(
+            rename_to="cum_buy",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "cumSell": ColumnSpec(
+            rename_to="cum_sell",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "cashFlow": ColumnSpec(rename_to="cash_flow", parse_dtype=pl.Float64),
+        "frozenCash": ColumnSpec(rename_to="frozen_cash", parse_dtype=pl.Float64),
+        "globalCumBuyNotional": ColumnSpec(
+            rename_to="cum_buy_filled_notional",
+            parse_dtype=pl.Float64,
+        ),
+        "globalCumSellNotional": ColumnSpec(
+            rename_to="cum_sell_filled_notional",
+            parse_dtype=pl.Float64,
+        ),
+    },
+    null_values=["", "NA", "null", "NULL"],
+    drop=["#HFTORD"],
+)
+
+
+# =============================================================================
+# JYAO Alpha Format (v2025-11-14)
+# =============================================================================
+
+JYAO_V20251114 = SchemaEvolution(
+    columns={
+        # Symbol column - parse_dtype for CSV, cast_dtype for feather/IPC
+        # (feather files have embedded types, so cast is needed post-load)
+        "ukey": ColumnSpec(parse_dtype=pl.Int64, cast_dtype=pl.Int64),
+        # Quote columns
+        "BidPrice1": ColumnSpec(rename_to="bid_px0", parse_dtype=pl.Float64),
+        "AskPrice1": ColumnSpec(rename_to="ask_px0", parse_dtype=pl.Float64),
+        "BidVolume1": ColumnSpec(
+            rename_to="bid_size0",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        "AskVolume1": ColumnSpec(
+            rename_to="ask_size0",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        # Time columns
+        "TimeStamp": ColumnSpec(rename_to="timestamp", parse_dtype=pl.Int64),
+        "GlobalExTime": ColumnSpec(rename_to="global_exchange_ts", parse_dtype=pl.Int64),
+        "DataDate": ColumnSpec(rename_to="data_date", parse_dtype=pl.String),
+        # Volume
+        "Volume": ColumnSpec(
+            rename_to="volume",
+            parse_dtype=pl.Float64,
+            cast_dtype=pl.Int64,
+        ),
+        # Predictor columns (x_* = alpha predictions)
+        # Rule: ≤60s → s, >60s → m
+        "x10s": ColumnSpec(rename_to="x_10s", parse_dtype=pl.Float64),
+        "x60s": ColumnSpec(rename_to="x_60s", parse_dtype=pl.Float64),
+        "alpha1": ColumnSpec(rename_to="x_3m", parse_dtype=pl.Float64),
+        "alpha2": ColumnSpec(rename_to="x_30m", parse_dtype=pl.Float64),
+    },
+    null_values=["", "NA"],
+)
+
+
+# =============================================================================
+# Schema Registry
+# =============================================================================
+
+SCHEMAS: dict[str, SchemaEvolution] = {
+    "ylin_v20251204": YLIN_V20251204,
+    "jyao_v20251114": JYAO_V20251114,
+}
+
+
+def get_schema(name: str | None) -> SchemaEvolution | None:
+    """Get SchemaEvolution by name.
+
+    Args:
+        name: Schema name (e.g., "ylin_v20251204") or None.
+
+    Returns:
+        SchemaEvolution or None if name is None or not found.
+    """
+    if not name:
+        return None
+    return SCHEMAS.get(name.lower())

{vizflow-0.5.1.dist-info → vizflow-0.5.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vizflow
-Version: 0.5.1
+Version: 0.5.3
 Requires-Python: >=3.9
 Requires-Dist: polars>=0.20.0
 Provides-Extra: dev

vizflow-0.5.3.dist-info/RECORD

@@ -0,0 +1,10 @@
+vizflow/__init__.py,sha256=pO3pTaKj7LNyuQlTrCHau9nBymhNGdmiImgJXH5lxt4,529
+vizflow/config.py,sha256=y4vRvjVTa1H5AdQf0q_XhYr-3EBDJst8BJq52ODN3uk,6456
+vizflow/io.py,sha256=eheqyLHGiSh69erxMk98FK-GYycbSheqkrIYRYGFy3A,7687
+vizflow/market.py,sha256=MtNz_nnZxC66Aq-i2PXEwaFCTknijFWYZUUv6798k2s,2493
+vizflow/ops.py,sha256=4UwxOTPhvZ1_4PI3pcxbXfLAYsn1Ecj6nyBtBBr7KS8,7761
+vizflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vizflow/schema_evolution.py,sha256=puwuuJ8LAzelHq1JTwQD968J7GYLvgAsCaHJnDHzu4U,14025
+vizflow-0.5.3.dist-info/METADATA,sha256=e8LTSGVMiHLNcTjMaAO11Lj5xYa6etsoG4yBuxI8E64,388
+vizflow-0.5.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+vizflow-0.5.3.dist-info/RECORD,,

vizflow/presets.py

@@ -1,87 +0,0 @@
-"""Column mapping presets for VizFlow."""
-
-# ylin's trade format (v2025-12-04)
-YLIN_V20251204 = {
-    # Order columns (18)
-    "symbol": "ukey",
-    "orderId": "order_id",
-    "orderSide": "order_side",
-    "orderQty": "order_qty",
-    "orderPrice": "order_price",
-    "priceType": "order_price_type",
-    "fillQty": "order_filled_qty",
-    "fillPrice": "fill_price",
-    "lastExchangeTs": "update_exchange_ts",
-    "createdTs": "create_exchange_ts",
-    "localTs": "create_local_ts",
-    "qtyAhead": "qty_ahead",
-    "qtyBehind": "qty_behind",
-    "orderStatus": "order_curr_state",
-    "orderTposType": "order_tpos_type",
-    "alphaTs": "alpha_ts",
-    "event": "event_type",
-    "cumFilledNotional": "order_filled_notional",
-    # Quote columns (15)
-    "bid": "bid_px0",
-    "bid2": "bid_px1",
-    "bid3": "bid_px2",
-    "bid4": "bid_px3",
-    "bid5": "bid_px4",
-    "ask": "ask_px0",
-    "ask2": "ask_px1",
-    "ask3": "ask_px2",
-    "ask4": "ask_px3",
-    "ask5": "ask_px4",
-    "bsize": "bid_size0",
-    "bsize2": "bid_size1",
-    "bsize3": "bid_size2",
-    "bsize4": "bid_size3",
-    "bsize5": "bid_size4",
-    "asize": "ask_size0",
-    "asize2": "ask_size1",
-    "asize3": "ask_size2",
-    "asize4": "ask_size3",
-    "asize5": "ask_size4",
-    "isRebasedQuote": "is_rebased",
-    "quoteSeqNum": "seq_num",
-    "quoteTs": "timestamp",
-    # Position columns (11)
-    "startPos": "init_net_pos",
-    "pos": "current_net_pos",
-    "realizedPos": "current_realized_net_pos",
-    "openBuyPos": "open_buy",
-    "openSellPos": "open_sell",
-    "cumBuy": "cum_buy",
-    "cumSell": "cum_sell",
-    "cashFlow": "cash_flow",
-    "frozenCash": "frozen_cash",
-    "globalCumBuyNotional": "cum_buy_filled_notional",
-    "globalCumSellNotional": "cum_sell_filled_notional",
-}
-
-# jyao's alpha format (v2025-11-14)
-JYAO_V20251114 = {
-    # Quote columns
-    "BidPrice1": "bid_px0",
-    "AskPrice1": "ask_px0",
-    "BidVolume1": "bid_size0",
-    "AskVolume1": "ask_size0",
-    # Time columns
-    "TimeStamp": "timestamp",
-    "GlobalExTime": "global_exchange_ts",
-    "DataDate": "data_date",
-    # Volume
-    "Volume": "volume",
-    # Predictor columns (x_* = alpha predictions)
-    # Rule: ≤60s → s, >60s → m
-    "x10s": "x_10s",
-    "x60s": "x_60s",
-    "alpha1": "x_3m",
-    "alpha2": "x_30m",
-}
-
-# Preset registry for dynamic lookup
-PRESETS: dict[str, dict[str, str]] = {
-    "ylin_v20251204": YLIN_V20251204,
-    "jyao_v20251114": JYAO_V20251114,
-}

vizflow-0.5.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-vizflow/__init__.py,sha256=EJ8qF4o2grf4aSochaasaaf0unyXV5yhrMs6rAhyp7k,496
-vizflow/config.py,sha256=JNW5-TshQ1v-Ft3-VV0JYJ5PdC3Yhgy4fW0AV0RWzkE,7322
-vizflow/io.py,sha256=ypiEuuPoHFKSt6VnhXcEI7u7dyVjKORunjqGpkFauXM,8877
-vizflow/market.py,sha256=MtNz_nnZxC66Aq-i2PXEwaFCTknijFWYZUUv6798k2s,2493
-vizflow/ops.py,sha256=4UwxOTPhvZ1_4PI3pcxbXfLAYsn1Ecj6nyBtBBr7KS8,7761
-vizflow/presets.py,sha256=h91NZoOH4YAx0bbsaNigECf9WOcWh1QZavguunWkaLE,2452
-vizflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vizflow-0.5.1.dist-info/METADATA,sha256=DHzwPBvYuj7Rc4BawcXD2juS5iR5UD1FnAxt3cgvpo4,388
-vizflow-0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-vizflow-0.5.1.dist-info/RECORD,,