tse-tick 0.3.0__py3-none-any.whl

tse_tick/__init__.py

@@ -0,0 +1,174 @@
+"""
+tse_tick — Tokyo Stock Exchange tick data processing for NEEDS data.
+
+Authors and contributions:
+    Kazumi Li         — Schema definitions, package architecture, maintainer
+    Masataka Hayashi  — Initial pandas-based prototype
+    Peter Romero      — Original concept and initial project design
+
+Developed at Keio University, Nakatsuma Seminar.
+"""
+
+# tse_tick/__init__.py
+import polars as pl
+
+__version__ = "0.3.0"
+__author__ = "Kazumi Li"
+__email__ = "kaiwenli@keio.jp"
+__license__ = "MIT"
+__copyright__ = "Copyright 2025-2026"
+
+from .enhanced import create_df, export_to_csv, discover_zips, parse_period, read_ticks
+
+from .schemas import (
+    get_schema_individual_stock_95,
+    get_schema_summary_83,
+    get_schema_indices_23,
+    get_schema_indices_summary,
+    get_japanese_column_mapping,
+)
+
+from .ingest import (
+    ingest_single_zip,
+    ingest_directory,
+    ingest_year,
+    ingest_year_from_root,
+    ingest_period,
+    ingest_event_windows_period,
+)
+
+from .io.parquet import (
+    write_partitioned_parquet,
+    read_parquet_partition,
+    write_event_window_parquet,
+    read_partitioned_parquet,
+)
+
+from .event_window import extract_event_window, extract_batch_event_windows
+
+from .features import (
+    compute_spread,
+    compute_depth,
+    compute_flow_imbalance,
+    compute_volatility,
+    compute_all_features,
+)
+
+from .constants import DataType, Language
+from .translate import translate, mapping
+
+try:
+    from .query import (
+        query_ticks,
+        query_sql,
+        get_available_dates,
+        get_available_tickers,
+    )
+    _DUCKDB_AVAILABLE = True
+except ImportError:
+    _DUCKDB_AVAILABLE = False
+
+    def _duckdb_unavailable(*args, **kwargs):
+        raise ImportError(
+            "DuckDB is required for query functions. Install it with: pip install duckdb>=0.9.0"
+        )
+
+    query_ticks = _duckdb_unavailable
+    query_sql = _duckdb_unavailable
+    get_available_dates = _duckdb_unavailable
+    get_available_tickers = _duckdb_unavailable
+
+__all__ = [
+    "create_df",
+    "read_ticks",
+    "export_to_csv",
+    "discover_zips",
+    "parse_period",
+    "get_schema_individual_stock_95",
+    "get_schema_summary_83",
+    "get_schema_indices_23",
+    "get_schema_indices_summary",
+    "get_japanese_column_mapping",
+    "ingest_single_zip",
+    "ingest_directory",
+    "ingest_year",
+    "ingest_year_from_root",
+    "ingest_period",
+    "ingest_event_windows_period",
+    "write_partitioned_parquet",
+    "read_parquet_partition",
+    "write_event_window_parquet",
+    "read_partitioned_parquet",
+    "query_ticks",
+    "query_sql",
+    "get_available_dates",
+    "get_available_tickers",
+    "extract_event_window",
+    "extract_batch_event_windows",
+    "compute_spread",
+    "compute_depth",
+    "compute_flow_imbalance",
+    "compute_volatility",
+    "compute_all_features",
+    "translate",
+    "mapping",
+    "DataType",
+    "Language",
+    "__version__",
+    "__author__",
+]
+
+
+def get_version():
+    return __version__
+
+
+def get_supported_data_types():
+    # Derive from the DataType enum so this list can never drift from it.
+    return DataType.values()
+
+
+def get_supported_years():
+    from datetime import datetime as _dt
+    return (2016, _dt.now().year)
+
+
+def get_info():
+    info = f"""
+    tse_tick v{__version__}
+    ========================
+
+    Author: {__author__}
+    License: {__license__}
+
+    Supported Data Types (output fields):
+    - individual_stock (TICST120) - 95 fields
+    - stock_summary (TICSS110) - 82 fields (83 raw)
+    - indices (TICIT110) - 10 fields (23 raw, 15 in 2016)
+    - indices_summary (TICIS110) - 17 fields
+
+    Year Range: 2016-2025
+
+    Languages: English (en), Japanese (jp)
+
+    Quick Start (two access paths):
+    # One-shot - read raw ZIPs straight to a filtered DataFrame (no store):
+    >>> import tse_tick
+    >>> df = tse_tick.read_ticks("DATA_ROOT", ticker_filter={{"7203"}},
+    ...                          date="20240201", start_time="09:00:00",
+    ...                          end_time="11:30:00")
+
+    # Two-stage - ingest once into a Parquet store, then query repeatedly:
+    >>> from tse_tick import DataType
+    >>> tse_tick.query_ticks(store, data_type=DataType.INDIVIDUAL_STOCK,
+    ...                      ticker=7203, date="20240201",
+    ...                      start_time="09:00:00", end_time="11:30:00")
+
+    CLI Usage:
+    >>> tse-tick ingest --data-type individual_stock --period 2024 \\
+                --input-root /path/to/data --output-root /path/to/store
+
+    For more information, visit:
+    https://github.com/tse-tick/tse_tick
+    """
+    print(info)

tse_tick/cli.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+# tse_tick/cli.py
+import argparse
+import logging
+import sys
+
+from tse_tick.ingest import ingest_directory, ingest_year_from_root, ingest_period, ingest_event_windows_period
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_years(years_str: str) -> list[int]:
+    result: list[int] = []
+    for part in years_str.split(","):
+        part = part.strip()
+        if "-" in part:
+            try:
+                start, end = part.split("-")
+                result.extend(range(int(start), int(end) + 1))
+            except ValueError:
+                raise ValueError(f"Invalid year range: {part}")
+        else:
+            try:
+                result.append(int(part))
+            except ValueError:
+                raise ValueError(f"Invalid year: {part}")
+    return sorted(set(result))
+
+
+def _parse_months(months_str: str) -> list[int]:
+    result: list[int] = []
+    for part in months_str.split(","):
+        part = part.strip()
+        if "-" in part:
+            try:
+                start, end = part.split("-")
+                result.extend(range(int(start), int(end) + 1))
+            except ValueError:
+                raise ValueError(f"Invalid month range: {part}")
+        else:
+            try:
+                result.append(int(part))
+            except ValueError:
+                raise ValueError(f"Invalid month: {part}")
+    return sorted(set(result))
+
+
+def _parse_tickers(tickers_str: str) -> set[str]:
+    ticker_str = tickers_str.strip()
+    if ticker_str.startswith("@"):
+        filepath = ticker_str[1:]
+        with open(filepath, "r") as f:
+            return {line.strip() for line in f if line.strip()}
+    return {t.strip() for t in ticker_str.split(",") if t.strip()}
+
+
+def cmd_ingest(args: argparse.Namespace) -> None:
+    valid_types = {"individual_stock", "stock_summary", "indices", "indices_summary"}
+    if args.data_type not in valid_types:
+        print(f"Error: --data-type must be one of {sorted(valid_types)}", file=sys.stderr)
+        sys.exit(1)
+
+    input_root = args.input_root
+    output_root = args.output_root
+
+    if args.filter_csv and args.data_type != "individual_stock":
+        print("Error: --filter-csv is only supported with --data-type individual_stock", file=sys.stderr)
+        sys.exit(1)
+
+    if args.period is not None:
+        mode_str = "full"
+        if args.filter_csv:
+            mode_str = "event-window"
+        elif args.tickers:
+            mode_str = f"ticker-filter ({len(_parse_tickers(args.tickers))} tickers)"
+        print(f"Ingesting by period: {args.period}  [{mode_str}]")
+        print(f"  Data type: {args.data_type}")
+        print(f"  Output: {output_root}")
+
+        if args.filter_csv:
+            ingest_event_windows_period(
+                input_root, output_root,
+                period=args.period,
+                filter_csv=args.filter_csv,
+                window_minutes=args.window,
+                resume=not args.no_resume,
+                max_workers=args.parallel,
+            )
+            print("Done")
+        else:
+            ticker_filter = _parse_tickers(args.tickers) if args.tickers else None
+            results = ingest_period(
+                input_root, output_root,
+                period=args.period,
+                data_type=args.data_type,
+                language=args.language,
+                resume=not args.no_resume,
+                max_workers=args.parallel,
+                ticker_filter=ticker_filter,
+            )
+            success = sum(1 for r in results if "error" not in r)
+            failed = sum(1 for r in results if "error" in r)
+            print(f"Done: {success} succeeded, {failed} failed")
+        return
+
+    years = _parse_years(args.years) if args.years else [args.year] if args.year else None
+    if years is None:
+        print("Error: --years, --year, or --period is required", file=sys.stderr)
+        sys.exit(1)
+
+    ticker_filter = _parse_tickers(args.tickers) if args.tickers else None
+
+    if args.flat:
+        print(f"Ingesting all ZIPs from flat directory: {input_root}")
+        print(f"  Data type: {args.data_type}")
+        print(f"  Output: {output_root}")
+        results = ingest_directory(
+            input_root, output_root,
+            data_type=args.data_type,
+            language=args.language,
+            max_workers=args.parallel,
+            ticker_filter=ticker_filter,
+        )
+        success = sum(1 for r in results if "error" not in r)
+        failed = sum(1 for r in results if "error" in r)
+        print(f"Done: {success} succeeded, {failed} failed")
+    else:
+        print(f"Ingesting from root with discovery: {input_root}")
+        print(f"  Data type: {args.data_type}")
+        print(f"  Years: {years}")
+        print(f"  Output: {output_root}")
+        for year in years:
+            print(f"\n--- Year {year} ---")
+            results = ingest_year_from_root(
+                input_root, output_root,
+                year=year,
+                data_type=args.data_type,
+                language=args.language,
+                resume=not args.no_resume,
+                ticker_filter=ticker_filter,
+            )
+            success = sum(1 for r in results if "error" not in r)
+            failed = sum(1 for r in results if "error" in r)
+            print(f"Year {year}: {success} succeeded, {failed} failed")
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="tse-tick",
+        description="High-performance Nikkei NEEDS tick data processing pipeline",
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    ingest_parser = subparsers.add_parser("ingest", help="Ingest NEEDS ZIP files into Parquet store")
+    ingest_parser.add_argument(
+        "--data-type",
+        required=True,
+        choices=["individual_stock", "stock_summary", "indices", "indices_summary"],
+        help="Type of NEEDS data to ingest",
+    )
+    ingest_parser.add_argument(
+        "--years",
+        type=str,
+        default=None,
+        help='Year(s) to process, e.g. "2016-2023" or "2018,2019,2020"',
+    )
+    ingest_parser.add_argument(
+        "--year",
+        type=int,
+        default=None,
+        help="Single year to process (alternative to --years)",
+    )
+    ingest_parser.add_argument(
+        "--period",
+        type=str,
+        default=None,
+        help=(
+            "Date range to process: YYYY (entire year), "
+            "YYYYMM-YYYYMM (month range), or YYYYMMDD-YYYYMMDD (day range). "
+            "Takes precedence over --years/--year."
+        ),
+    )
+    ingest_parser.add_argument(
+        "--input-root",
+        required=True,
+        help="Root directory containing NEEDS data in {year}/{yearmonth}/ layout",
+    )
+    ingest_parser.add_argument(
+        "--output-root",
+        required=True,
+        help="Root directory for Parquet output store",
+    )
+    ingest_parser.add_argument(
+        "--language",
+        default="en",
+        choices=["en", "jp"],
+        help="Column name language (default: en)",
+    )
+    ingest_parser.add_argument(
+        "--parallel",
+        type=int,
+        default=1,
+        help="Number of parallel worker processes (default: 1)",
+    )
+    ingest_parser.add_argument(
+        "--no-resume",
+        action="store_true",
+        help="Disable resume (reprocess all files even if output exists)",
+    )
+    ingest_parser.add_argument(
+        "--flat",
+        action="store_true",
+        help="Input directory is a flat folder of ZIPs (no year/month structure)",
+    )
+    ingest_parser.add_argument(
+        "--log-level",
+        default="INFO",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        help="Logging verbosity level",
+    )
+    ingest_parser.add_argument(
+        "--tickers",
+        type=str,
+        default=None,
+        help=(
+            "Comma-separated ticker codes to keep, or @file.txt with one per line. "
+            "When provided, only these tickers are included in output."
+        ),
+    )
+    ingest_parser.add_argument(
+        "--filter-csv",
+        type=str,
+        default=None,
+        help="Path to event filter CSV (enables event-window mode). Overrides --tickers.",
+    )
+    ingest_parser.add_argument(
+        "--window",
+        type=int,
+        default=120,
+        help="Window in minutes around each event (default: 120). Only used with --filter-csv.",
+    )
+
+    return parser
+
+
+def main() -> None:
+    parser = _build_parser()
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        sys.exit(1)
+
+    logging.basicConfig(
+        level=getattr(logging, args.log_level, logging.INFO),
+        format="%(asctime)s  %(levelname)-8s  %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    if args.command == "ingest":
+        cmd_ingest(args)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tse_tick/constants.py

@@ -0,0 +1,51 @@
+# tse_tick/constants.py
+"""Enumerations for the public API.
+
+``DataType`` and ``Language`` give IDE autocomplete and a single source of truth
+for the magic strings used throughout the package. Both subclass ``str``, so a
+member is accepted anywhere the equivalent string is today — comparisons,
+dict/set membership, ``pathlib`` path building and f-strings all see the value::
+
+    >>> from tse_tick import DataType
+    >>> DataType.INDIVIDUAL_STOCK == "individual_stock"
+    True
+    >>> f"{DataType.INDIVIDUAL_STOCK}"
+    'individual_stock'
+"""
+
+from enum import Enum
+from typing import List
+
+
+class DataType(str, Enum):
+    """NEEDS data type. The value is the canonical string used across the API."""
+
+    INDIVIDUAL_STOCK = "individual_stock"
+    STOCK_SUMMARY = "stock_summary"
+    INDICES = "indices"
+    INDICES_SUMMARY = "indices_summary"
+
+    def __str__(self) -> str:
+        # Return the bare value (not "DataType.INDIVIDUAL_STOCK") so f-strings and
+        # str() match the legacy magic-string behaviour on every Python version.
+        return self.value
+
+    @classmethod
+    def values(cls) -> List[str]:
+        """All data-type values as plain strings, in declaration order."""
+        return [member.value for member in cls]
+
+
+class Language(str, Enum):
+    """Output column-name language."""
+
+    EN = "en"
+    JP = "jp"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @classmethod
+    def values(cls) -> List[str]:
+        """All language values as plain strings, in declaration order."""
+        return [member.value for member in cls]