msd-backtest 0.1.2__tar.gz → 0.1.4__tar.gz

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: msd-backtest
-Version: 0.1.2
+Version: 0.1.4
 Summary: MSD backtest framework
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -10,7 +10,8 @@ Requires-Dist: py-alpha-lib>=0.2; extra == "alpha"
 Provides-Extra: msd
 Requires-Dist: pymsd>=0.1; extra == "msd"
 Requires-Dist: requests>=2.32; extra == "msd"
-Requires-Dist: pandas>=3.0; extra == "msd"
+Requires-Dist: polars>=1.0.0; extra == "msd"
+Requires-Dist: xlsxwriter>=3.2.9; extra == "msd"
 Provides-Extra: server
 Requires-Dist: fastapi[standard]>=0.135.1; extra == "server"
 Requires-Dist: msd-backtest[cli]; extra == "server"

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/mbt/__init__.py

@@ -9,20 +9,23 @@ from .account import (
   ACTION_KEEP,
 )
 from .run.runner import Runner
+from .select import Selector, SelectorDataProvider
 
 
 __all__ = [
-  "Context",
-  "Strategy",
-  "Runner",
-  "Account",
-  "Fee",
-  "DataProvider",
-  "Operation",
-  "Position",
   "ACTION_BUY",
   "ACTION_BUY_NO_CASH",
+  "ACTION_KEEP",
   "ACTION_SELL",
   "ACTION_SELL_NO_HOLD",
-  "ACTION_KEEP",
+  "Account",
+  "Context",
+  "DataProvider",
+  "Fee",
+  "Operation",
+  "Position",
+  "Runner",
+  "Selector", 
+  "SelectorDataProvider",
+  "Strategy",
 ]

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/mbt/run/cli.py

@@ -9,7 +9,7 @@ from mbt.run.msd import load_data
 import logging
 from typing import Tuple
 import argparse
-import pandas as pd
+import polars as pl
 import orjson
 import os
 import numpy as np
@@ -111,7 +111,9 @@ def parse_args() -> Tuple[RunRequest, str, str, Fee]:
   return req, args.msd, args.output, fee
 
 
-def float_format(f: float) -> str:
+def float_format(f: float | None) -> str:
+  if f is None or (isinstance(f, float) and np.isnan(f)):
+    return ""
   s = f"{f:.6f}"
   if "." in s:
     s = s.rstrip("0").rstrip(".")
@@ -134,24 +136,26 @@ def build_output(reports: list[Report], output: str):
   logger.info(f"output kind: {ext}")
   if ext == "csv":
     for report in reports:
-      df = pd.DataFrame(report.detailed(with_symbol=True))
-      df.to_csv(output, index=False, float_format=float_format)  # type: ignore
+      df = pl.DataFrame(report.detailed(with_symbol=True))
+      float_cols = [col for col, dtype in zip(df.columns, df.dtypes) if dtype in (pl.Float32, pl.Float64)]
+      df = df.with_columns([
+        pl.col(col).map_elements(float_format, return_dtype=pl.String)
+        for col in float_cols
+      ])
+      df.write_csv(output)
   elif ext == "xlsx":
-    with pd.ExcelWriter(output) as writer:
-      df_metrics = pd.DataFrame([r.metrics(True) for r in reports])
-      df_metrics.to_excel(writer, sheet_name="metrics", index=False)
-      for report in reports:
-        data = report.detailed()
-        data['price'] = report.price
-        data['fw_price'] = report.fw_price
-        data['bw_price'] = report.bw_price
-        df = pd.DataFrame(data)
-        df.to_excel(
-          writer,
-          sheet_name=report.symbol,
-          index=False,
-          float_format="%.6f",
-        )
+    import xlsxwriter
+    wb = xlsxwriter.Workbook(output)
+    df_metrics = pl.DataFrame([r.metrics(True) for r in reports])
+    df_metrics.write_excel(workbook=wb, worksheet="metrics")
+    for report in reports:
+      data = report.detailed()
+      data['price'] = report.price
+      data['fw_price'] = report.fw_price
+      data['bw_price'] = report.bw_price
+      df = pl.DataFrame(data)
+      df.write_excel(workbook=wb, worksheet=report.symbol)
+    wb.close()
   elif ext == "json":
     data = build_json_report(reports)
     logger.info("build json")
@@ -163,8 +167,8 @@ def build_output(reports: list[Report], output: str):
 
 
 def echo_reports(reports: list[Report]):
-  df_metrics = pd.DataFrame([r.metrics(True) for r in reports])
-  print(df_metrics.to_json(orient='records', indent=2))
+  metrics_list = [r.metrics(True) for r in reports]
+  print(orjson.dumps(metrics_list, option=orjson.OPT_INDENT_2 | orjson.OPT_SERIALIZE_NUMPY).decode())
 
 
 def main():

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/mbt/run/msd.py

@@ -28,7 +28,7 @@ def load_data(
     DataProvider and symbols
   """
 
-  client = pymsd.create_msd_pandas(msd_host)
+  client = pymsd.create_msd_polars(msd_host)
 
   dfs = client.load(
     objs=symbols,
@@ -45,6 +45,9 @@ def load_data(
     return DataProvider({"ts": np.array([])}, symbols=[]), []
 
   data = client.adaptor.to_numpy(data)
+  for col in ["dividend", "transfer_shares", "right_shares", "right_price"]:
+    if col not in data:
+      data[col] = np.zeros_like(data["close"])
 
   dp = DataProvider(data, symbols=symbols, next_hook=next_hook_al)
 

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/mbt/run/report.py

@@ -1,6 +1,6 @@
 from mbt import Account
 from mbt import DataProvider
-import pandas as pd
+import polars as pl
 import numpy as np
 import dataclasses
 
@@ -49,9 +49,9 @@ class Report:
     self.ror_original = np.round(uniform_ror(dp.slice("bw_price", s)), 4)
     
     # Clean NaNs by forward filling and filling leading NaNs with 0
-    self.ror = pd.Series(self.ror).ffill().fillna(0.0).values
-    self.ror_hold = pd.Series(self.ror_hold).ffill().fillna(0.0).values
-    self.ror_original = pd.Series(self.ror_original).ffill().fillna(0.0).values
+    self.ror = pl.Series("", self.ror).fill_nan(None).fill_null(strategy="forward").fill_null(0.0).to_numpy()
+    self.ror_hold = pl.Series("", self.ror_hold).fill_nan(None).fill_null(strategy="forward").fill_null(0.0).to_numpy()
+    self.ror_original = pl.Series("", self.ror_original).fill_nan(None).fill_null(strategy="forward").fill_null(0.0).to_numpy()
 
     self.actions = dp.slice("actions", s)
     self.price = dp.slice("price", s)

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/mbt/run/review.py

@@ -2,7 +2,7 @@ from mbt.run.report import Report
 import orjson
 import sys
 import argparse
-import pandas as pd
+import polars as pl
 
 
 
@@ -29,10 +29,17 @@ def parse_args():
 
 def do_stat(args):
   reports = load_reports(args.report)
-  df = pd.DataFrame([r.metrics(True) for r in reports.values()])
-  df.set_index("symbol", inplace=True)
+  df = pl.DataFrame([r.metrics(True) for r in reports.values()])
+  if "symbol" in df.columns:
+    df = df.drop("symbol")
   stat = df.describe()
-  print(stat.to_json())
+  stat_col = stat.columns[0]
+  stats = stat[stat_col].to_list()
+  res = {}
+  for col in stat.columns[1:]:
+    vals = stat[col].to_list()
+    res[col] = {s: v for s, v in zip(stats, vals) if s != "null_count" and v is not None}
+  print(orjson.dumps(res).decode())
 
 def do_detail(args):
   reports = load_reports(args.report)
@@ -44,12 +51,12 @@ def do_detail(args):
 
 def do_sort(args):
   reports = load_reports(args.report)
-  df = pd.DataFrame([{"symbol":r.symbol, args.metric: getattr(r, args.metric, 0)} for r in reports.values()])
-  df.set_index("symbol", inplace=True)
-  df = df.sort_values(by=args.metric, ascending=args.order == "asc")
+  records = [{"symbol": r.symbol, "val": getattr(r, args.metric, 0)} for r in reports.values()]
+  records.sort(key=lambda x: x["val"], reverse=args.order != "asc")
   if args.limit:
-    df = df.head(args.limit)
-  print(df.to_json(orient="columns"))
+    records = records[:args.limit]
+  res = {args.metric: {r["symbol"]: r["val"] for r in records}}
+  print(orjson.dumps(res).decode())
   
 def load_reports(report_path: str) -> dict[str, Report]:
   with open(report_path, "r") as f:

msd_backtest-0.1.4/mbt/select/__init__.py

@@ -0,0 +1,18 @@
+from .selector import Selector, SelectorDataProvider
+from .msd import MsdSelectorDataProvider
+from .stocks import ALL_STOCKS, A_STOCKS, A_STOCKS_EXCLUDE_ST, FOUNDS, KIND_INDEX, KIND_FUND, KIND_STOCK, STATUS_NORMAL, STATUS_ST, STATUS_STAR_ST
+
+__all__ = [
+  'ALL_STOCKS',
+  'A_STOCKS',
+  'A_STOCKS_EXCLUDE_ST',
+  'FOUNDS',
+  'KIND_INDEX',
+  'KIND_FUND',
+  'STATUS_NORMAL',
+  'STATUS_ST',
+  'STATUS_STAR_ST',
+  'Selector',
+  'SelectorDataProvider',
+  'MsdSelectorDataProvider',
+]

msd_backtest-0.1.4/mbt/select/cli.py

@@ -0,0 +1,152 @@
+import logging
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+
+from typing import Any
+from mbt.select import Selector, SelectorDataProvider, MsdSelectorDataProvider
+import mbt.select.stocks 
+import logging
+from typing import Tuple
+import argparse
+import os
+import sys
+import importlib.util
+
+logging.basicConfig(
+  format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.INFO
+)
+
+logger = logging.getLogger("selector")
+logger.setLevel(logging.DEBUG)
+
+
+def expand_symbols(symbols: list[str]) -> list[str]:
+  """read as file when input is a file path, and expand it. otherwise, return the input as a list of symbols"""
+  expanded = set()
+  for s in symbols:
+    if os.path.isfile(s):
+      with open(s, "r") as f:
+        lines = filter(
+          lambda x: len(x) > 0 and not x.startswith("#"),
+          map(lambda x: x.strip(), f.readlines()),
+        )
+        expanded.update(lines)
+    else:
+      match s:
+        case "A_STOCKS_EXCLUDE_ST":
+          expanded.update(mbt.select.stocks.A_STOCKS_EXCLUDE_ST)
+        case "A_STOCKS":
+          expanded.update(mbt.select.stocks.A_STOCKS)
+        case "FOUNDS":
+          expanded.update(mbt.select.stocks.FOUNDS)
+        case _:
+          expanded.add(s)
+
+  l = list(expanded)
+  l.sort()
+  return l
+
+
+def load_selector(py_file: str, **kwargs) -> Selector:
+  path = os.path.abspath(py_file)
+  dir_path = os.path.dirname(path)
+  if dir_path not in sys.path:
+    sys.path.insert(0, dir_path)
+
+  module_name = os.path.splitext(os.path.basename(path))[0]
+  spec = importlib.util.spec_from_file_location(module_name, path)
+  if spec and spec.loader:
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    for obj in vars(module).values():
+      if isinstance(obj, type) and issubclass(obj, Selector) and obj is not Selector:
+        return obj(**kwargs) if kwargs is not None else obj()
+  raise ValueError(f"Could not find a Selector class in {py_file}")
+
+
+
+def parse_args(args: list[str]) -> dict[str, Any]:
+  if args is None:
+    return {}
+  kwargs = {}
+
+  def try_bool(s: str) -> Tuple[bool, bool]:
+    match s.lower():
+      case "true" | "t" | "yes" | "y" | "on":
+        return True, True
+      case "false" | "f" | "no" | "n" | "off":
+        return False, True
+      case _:
+        return False, False
+    
+  def try_int(s: str) -> Tuple[int, bool]:
+    try:
+      return int(s), True
+    except ValueError:
+      return 0, False
+
+  def try_float(s: str) -> Tuple[float, bool]:
+    try:
+      return float(s), True
+    except ValueError:
+      return 0.0, False
+    
+  def try_float(s: str) -> Tuple[float, bool]:
+    try:
+      return float(s), True
+    except ValueError:
+      return 0.0, False
+    
+  def try_value(s: str) -> Any:
+    for try_func in [try_bool, try_int, try_float]:
+      v, ok = try_func(s)
+      if ok:
+        return v
+    return s
+
+  for arg in args:
+    kv = arg.split(arg, 1)
+    if len(kv) != 2:
+      logger.warning(f'invalid arg "{arg}"')
+      continue
+    key, value = kv
+    kwargs[key] = try_value(value)
+  return kwargs
+
+def main():
+  parser = argparse.ArgumentParser(description="Run a selector")
+  parser.add_argument("selector", help="Selector Python file")
+  parser.add_argument("-a", "--arg", nargs='*', help="any init arguments with format key=value")
+  parser.add_argument(
+    "-m", "--msd", type=str, default=os.environ.get("MSD_HOST", "http://localhost:50510"), help="MSD host, env $MSD_HOST"
+  )
+  parser.add_argument("-s", "--symbols", nargs="+", default=[], help="Symbols to run")
+  args = parser.parse_args()
+  kwargs = parse_args(args.arg)
+  selector = load_selector(args.selector, **kwargs)
+  logger.info(f"selector loaded: {selector}")
+
+  dp = MsdSelectorDataProvider(args.msd)
+  logger.info("data provider created")
+
+  init_symbols = expand_symbols(args.symbols)
+
+  stocks = selector.execute(dp, init_symbols)
+  logger.info("selector finished")
+  factors = getattr(selector, 'factors', None)
+  if factors is not None:
+    print(factors)
+  else:
+    for stock in stocks:
+      print(stock)
+
+if __name__ == "__main__":
+  main()
+  
+
+
+
+
+
+

msd_backtest-0.1.4/mbt/select/msd.py

@@ -0,0 +1,124 @@
+import pymsd
+from .selector import SelectorDataProvider
+import polars as pl
+import alpha as al
+import pymsd
+import logging
+
+logger = logging.getLogger("selector")
+
+
+class MsdSelectorDataProvider(SelectorDataProvider):
+
+  def __init__(self, msd_host: str):
+    super().__init__()
+    self._msd_host = msd_host
+    self._client = pymsd.create_msd_polars(msd_host)
+
+
+  def load_kline(self, symbols: list[str], lastN: int = 100) -> dict[str, pl.DataFrame]:
+    if lastN == 1:
+      dividend_lastN = 1
+      shares_lastN = 1
+    else:
+      dividend_lastN = max(lastN//20, 5)
+      shares_lastN = max(lastN//10, 5)
+
+    logger.debug(f"start load kline for {len(symbols)} symbols, lastN={lastN}")
+    dfs = self._client.load(
+      objs=symbols,
+      tables=["stock_kline_1d", "stock_dividend", "stock_shares"],
+      join={"stock_dividend": "zero", "*": "backward"},
+      start=[lastN, dividend_lastN, shares_lastN],
+      end=self.date,
+    )
+    logger.debug(f"finish load kline for {len(symbols)} symbols")
+
+    
+    # Apply forward adjustment factor to price related columns
+    al.set_ctx(groups=1, flags=al.FLAG_SKIP_NAN)
+    for symbol, data in dfs.items():
+      if len(data["ts"]) == 1:
+        # Skip if there is only one day of data
+        continue
+      dividends = data["dividend"].to_numpy()
+      transfer_shares = data["transfer_shares"].to_numpy()
+      right_shares = data["right_shares"].to_numpy()
+      right_price = data["right_price"].to_numpy()
+      dfs[symbol] = data.with_columns(
+        close=al.FW_SPLIT(data["close"].to_numpy(), dividends, transfer_shares, right_shares, right_price),
+        open=al.FW_SPLIT(data["open"].to_numpy(), dividends, transfer_shares, right_shares, right_price),
+        high=al.FW_SPLIT(data["high"].to_numpy(), dividends, transfer_shares, right_shares, right_price),
+        low=al.FW_SPLIT(data["low"].to_numpy(), dividends, transfer_shares, right_shares, right_price)
+      )
+
+    return dfs
+    
+  def load_financial(self, symbols: list[str], fields: list[str], only_year: bool = True, lastN: int = 3) -> dict[str, pl.DataFrame]:
+    if only_year:
+      lastN = lastN * 4 + 1
+    
+    logger.debug(f"start load financial for {len(symbols)} symbols, lastN={lastN}")
+    dfs = self._client.load(
+      objs=symbols,
+      tables=["stock_financial"],
+      start=lastN,
+      end=self.date,
+      fields=fields,
+    )
+    logger.debug(f"finish load financial for {len(symbols)} symbols")
+
+    data = {}
+    for symbol, v in dfs.items():
+      df = v['stock_financial']
+      # df = df.with_columns(
+      #   pl.col("ts").map_elements(lambda d: int(d.year), return_dtype=pl.Int32).alias("f_year"),
+      #   pl.col("ts").map_elements(lambda d: int(d.month/3), return_dtype=pl.Int32).alias("f_quarter")
+      # )
+      if only_year:
+        data[symbol] = df.filter(pl.col('ts').dt.month() == 12)
+      else:
+        data[symbol] = df
+    return data
+
+  def load_snapshot(self, symbols: list[str], fin_fields: list[str] = [], fin_only_year: bool = True) -> pl.DataFrame:
+
+    dfs = self._client.load(
+      objs=symbols,
+      tables=["stock_kline_1d", "stock_shares"],
+      join={"stock_shares": "backward"},
+      start=1,
+      end=self.date
+    )
+
+    if len(fin_fields) > 0:
+      for obj, fin_df in self.load_financial(symbols, fin_fields, fin_only_year, 1).items():
+        if obj in dfs:
+          if len(fin_df) > 1:
+            fin_df = fin_df.tail(1)
+          dfs[obj] = pl.concat([dfs[obj], fin_df.rename({"ts": "f_ts"})], how='horizontal_extend')
+
+    return pl.concat([
+      df.with_columns(
+        pl.lit(obj).alias('obj')
+      ) for obj, df in dfs.items()
+    ])
+
+
+if __name__ == "__main__":
+  import os
+  import logging
+  from .stocks import A_STOCKS_EXCLUDE_ST
+  logging.basicConfig(level=logging.INFO)
+  msd_host = os.environ.get("MSD_HOST", "http://localhost:50511") 
+  if not msd_host:
+    raise Exception("MSD_HOST is not set")
+  dp = MsdSelectorDataProvider(msd_host)
+  klines = dp.load_kline(["SH600000"], 200)
+  financial = dp.load_financial(["SH600000"], ["f001", "f007"], False, 12)
+  #print(klines)
+  print(financial)
+
+  snap = dp.load_snapshot(A_STOCKS_EXCLUDE_ST, ["f137"])
+  snap.write_csv("dev/snap.csv")
+  

msd_backtest-0.1.4/mbt/select/selector.py

@@ -0,0 +1,89 @@
+from uvicorn import logging
+import polars as pl
+import logging
+import time
+
+logger = logging.getLogger("selector")
+
+
+class SelectorDataProvider:
+
+  date: str|None = None
+
+  """
+  Provide data to Selector
+  """
+
+  def load_kline(self, symbols: list[str], lastN: int = 100) -> dict[str, pl.DataFrame]:
+    """
+    load `lastN` kline bars for `symbols`
+
+    Returns:
+      A dict of [symbol] -> pl.DataFrame of lastN kline bars
+      DataFrame must have columns 
+        ['ts', 'open', 'high', 'low', 'close', 'amount', 'volume', 'total_shares', 'tradable_shares']
+      All prices had been forward-adjusted (split, bonus, etc. already reflected).
+      """
+    pass
+
+  def load_financial(self, symbols: list[str], fields: list[str], only_year: bool = True, lastN: int = 12) -> dict[str, pl.DataFrame]:
+    """
+    load `lastN` financial data for `symbols`
+    Financial data is provided on a quarterly basis, with dates of 03-30, 06-30, 09-30, and 12-31. 
+    When the only_year parameter is set to true, only the annual data (12-31) is selected.
+
+    Returns:
+      A dict of [symbol] -> pl.DataFrame of lastN financial data
+      DataFrame columns ['ts', *fields] 
+    """
+    pass
+
+  def load_snapshot(self, symbols: list[str], fin_fields: list[str] = [], fin_only_year: bool = True) -> pl.DataFrame:
+    """
+    Take a snapshot for the given symbols at 'snapshot_date', this is more faster than 
+    load_kline and load_financial separately.
+
+    User usually use this function to get the latest data for symbols.
+
+    Args:
+      symbols: symbols to snapshot
+      fin_fields: fields to snapshot from financial data
+      fin_only_year: whether to only snapshot annual financial data
+
+    Returns:
+      A DataFrame of snapshot for the given symbols, with columns combined from 
+      `load_kline` and `load_financial` except the financial data's 'ts` column will be renamed to 'f_ts'
+    """
+    pass
+
+
+class Selector:
+
+  dp: SelectorDataProvider
+
+  def __init__(self) :
+    pass
+
+  def execute(self, dp: SelectorDataProvider, init_stocks: list[str]) -> list[str]:
+    self.dp = dp
+    stocks = init_stocks
+    t0 = time.time()
+    logger.info(f"start execute selector, {self.__class__.__name__}")
+    for i in range(100):
+      fn_name = f'step{i:02d}'
+      fn = getattr(self, fn_name, None)
+      if fn is None:
+        break
+      t1 = time.time()
+      logger.info(f"start {fn_name} with {len(stocks)} stocks")
+      if len(fn.__doc__.strip()) > 0:
+        logger.info(fn.__doc__.strip())
+      stocks = fn(stocks)
+      logger.info(f"done {fn_name}, used {time.time() - t1:0.3f}s, remain {len(stocks)}")
+    logger.info(f"done execute selector {self.__class__.__name__}, used {time.time() - t0:0.3f}s, remain {len(stocks)}")
+    return stocks
+
+    
+
+
+

msd_backtest-0.1.4/mbt/select/stocks.py

@@ -0,0 +1,75 @@
+import orjson
+import polars as pl
+import os
+
+
+
+KIND_STOCK = 1
+KIND_FUND = 2
+KIND_INDEX = 3
+
+STATUS_NORMAL = 0
+STATUS_ST = 1
+STATUS_STAR_ST = 2
+
+def parse_obj_kind(kind: str) -> int:
+  match kind.lower():
+    case 'stock':
+      return KIND_STOCK
+    case 'fund':
+      return KIND_FUND
+    case 'index':
+      return KIND_INDEX
+  return 0
+
+
+def parse_obj_status(name: str) -> int:
+  if 'ST' in name:
+    return 1
+  if '*ST' in name:
+    return 2
+  return 0
+
+
+def load_all_stocks(path: str) -> pl.DataFrame:
+  with open(path) as fp:
+    content = orjson.loads(fp.read())
+    df = pl.DataFrame(list(content.values()), schema={
+      'symbol': pl.String,
+      'name': pl.String,
+      "volume_unit": pl.UInt8,
+      "price_decimal": pl.String,
+      'kind': pl.String,
+    })
+
+  df = df.with_columns([
+    pl.col('kind').map_elements(parse_obj_kind, return_dtype=pl.Int8),
+    pl.col('name').map_elements(parse_obj_status, return_dtype=pl.Int8).alias("status"),
+  ])
+  return df
+
+
+
+def where_clause(symbols: list[str], col: str = 'obj') -> str:
+  sql = ','.join([f'"{s}"' for s in symbols])
+  sql = f"{col} IN ({sql})"
+  return sql
+
+ALL_STOCKS = load_all_stocks(os.environ.get("MSD_STOCKS_FILE", 'etc/stocks.json'))
+A_STOCKS = ALL_STOCKS.filter(pl.col("kind") == KIND_STOCK).get_column("symbol").to_list()
+A_STOCKS_EXCLUDE_ST = ALL_STOCKS.filter(
+  (pl.col("kind") == KIND_STOCK) &
+  (pl.col("status") == STATUS_NORMAL)
+).get_column("symbol").to_list()
+FOUNDS = ALL_STOCKS.filter(pl.col("kind") == KIND_FUND).get_column("symbol").to_list()
+
+
+if __name__ == "__main__":
+  print(ALL_STOCKS)
+  print(len(A_STOCKS))
+  print(len(FOUNDS))
+  print(len(A_STOCKS_EXCLUDE_ST))
+
+  with open("a.txt", "w") as fp:
+    for symbol in A_STOCKS_EXCLUDE_ST:
+      fp.write(f"{symbol}\n")

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/msd_backtest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: msd-backtest
-Version: 0.1.2
+Version: 0.1.4
 Summary: MSD backtest framework
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -10,7 +10,8 @@ Requires-Dist: py-alpha-lib>=0.2; extra == "alpha"
 Provides-Extra: msd
 Requires-Dist: pymsd>=0.1; extra == "msd"
 Requires-Dist: requests>=2.32; extra == "msd"
-Requires-Dist: pandas>=3.0; extra == "msd"
+Requires-Dist: polars>=1.0.0; extra == "msd"
+Requires-Dist: xlsxwriter>=3.2.9; extra == "msd"
 Provides-Extra: server
 Requires-Dist: fastapi[standard]>=0.135.1; extra == "server"
 Requires-Dist: msd-backtest[cli]; extra == "server"

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/msd_backtest.egg-info/SOURCES.txt

@@ -13,6 +13,10 @@ mbt/run/review.py
 mbt/run/runner.py
 mbt/run/server.py
 mbt/select/__init__.py
+mbt/select/cli.py
+mbt/select/msd.py
+mbt/select/selector.py
+mbt/select/stocks.py
 msd_backtest.egg-info/PKG-INFO
 msd_backtest.egg-info/SOURCES.txt
 msd_backtest.egg-info/dependency_links.txt
@@ -23,5 +27,6 @@ tests/test_alpha.py
 tests/test_load_strategy.py
 tests/test_report.py
 tests/test_runner.py
+tests/test_selector.py
 tests/test_strategy.py
 tests/test_strategy_loading_complex.py

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/msd_backtest.egg-info/requires.txt

@@ -12,7 +12,8 @@ tabulate>=0.10.0
 [msd]
 pymsd>=0.1
 requests>=2.32
-pandas>=3.0
+polars>=1.0.0
+xlsxwriter>=3.2.9
 
 [server]
 fastapi[standard]>=0.135.1

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "msd-backtest"
-version = "0.1.2"
+version = "0.1.4"
 description = "MSD backtest framework"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -12,7 +12,7 @@ excel = ["openpyxl>=3.1.5"]
 
 [project.optional-dependencies]
 alpha = ["py-alpha-lib>=0.2"]
-msd = ["pymsd>=0.1", "requests>=2.32", "pandas>=3.0"]
+msd = ["pymsd>=0.1", "requests>=2.32", "polars>=1.0.0", "xlsxwriter>=3.2.9"]
 server = ["fastapi[standard]>=0.135.1", "msd-backtest[cli]"]
 cli = [
     "orjson>=3.11.9",
@@ -33,6 +33,6 @@ addopts = "--benchmark-skip"
 [tool.setuptools.packages.find]
 include = ["mbt*"]
 
-# [tool.uv.sources]
-# py-alpha-lib = { path = "../alpha-lib-rs/py-binding", editable = true }
-# pymsd = { path = "../msd-rs2/bindings/python", editable = true }
+[tool.uv.sources]
+py-alpha-lib = { path = "../alpha-lib-rs/py-binding", editable = true }
+pymsd = { path = "../msd-rs2/bindings/python", editable = true }

{msd_backtest-0.1.2 → msd_backtest-0.1.4}/tests/test_alpha.py

@@ -22,7 +22,7 @@ def load_data_msd(
   msd_host: str, symbols: list[str], start: str, end: str
 ) -> tuple[DataProvider, list[str]]:
 
-  client = pymsd.create_msd_pandas(msd_host)
+  client = pymsd.create_msd_polars(msd_host)
 
   dfs = client.load(
     objs=symbols,

msd_backtest-0.1.4/tests/test_selector.py

@@ -0,0 +1,140 @@
+from datetime import datetime, timedelta
+import polars as pl
+import numpy as np
+import mbt
+from mbt.select import SelectorDataProvider
+from strategy.growth_large_cap_ma15_selector import GrowthLargeCapMA15Selector
+
+class MockSelectorDataProvider(SelectorDataProvider):
+  def __init__(self, kline_data, financial_data):
+    super().__init__()
+    self.kline_data = kline_data
+    self.financial_data = financial_data
+
+  def load_kline(self, symbols: list[str], lastN: int = 100) -> dict[str, pl.DataFrame]:
+    return {sym: self.kline_data[sym] for sym in symbols if sym in self.kline_data}
+
+  def load_financial(self, symbols: list[str], fields: list[str], only_year: bool = True, lastN: int = 12) -> dict[str, pl.DataFrame]:
+    return {sym: self.financial_data[sym] for sym in symbols if sym in self.financial_data}
+
+  def load_snapshot(self, symbols: list[str], fin_fields: list[str] = [], fin_only_year: bool = True, snapshot_date: str = '') -> pl.DataFrame:
+    rows = []
+    for sym in symbols:
+      if sym not in self.kline_data:
+        continue
+      k_df = self.kline_data[sym]
+      fin_df = self.financial_data.get(sym)
+      
+      close = k_df["close"][-1]
+      total_shares = k_df["total_shares"][-1]
+      row_dict = {
+        "obj": sym,
+        "close": close,
+        "total_shares": total_shares,
+      }
+      if fin_df is not None and len(fin_df) > 0:
+        fin_last = fin_df.tail(1)
+        for f in fin_fields:
+          if f in fin_last.columns:
+            row_dict[f] = fin_last[f][0]
+      rows.append(row_dict)
+    
+    if not rows:
+      schema = {
+        "obj": pl.String,
+        "close": pl.Float64,
+        "total_shares": pl.Float64,
+      }
+      for f in fin_fields:
+        schema[f] = pl.Float64
+      return pl.DataFrame([], schema=schema)
+      
+    return pl.DataFrame(rows)
+
+def test_growth_large_cap_ma15_selector():
+  # Define dates
+  dates_fin = [datetime(2023, 12, 31), datetime(2024, 12, 31), datetime(2025, 12, 31)]
+  end_date = datetime(2026, 3, 6)
+  dates_k = [end_date - timedelta(days=i) for i in range(19, -1, -1)]
+
+  # 1. SZ000001: All filters pass
+  # Net Profit: 100 -> 120 -> 150 (growing & positive)
+  df_fin_1 = pl.DataFrame({"ts": dates_fin, "f137": [100.0, 120.0, 150.0]})
+  # Close price drops below MA15 on the last day
+  close_1 = np.ones(20) * 12.0
+  close_1[-2] = 13.0
+  close_1[-1] = 8.0
+  df_k_1 = pl.DataFrame({
+    "ts": dates_k,
+    "open": close_1,
+    "high": close_1,
+    "low": close_1,
+    "close": close_1,
+    "volume": np.ones(20) * 1000,
+    "amount": np.ones(20) * 12000,
+    "total_shares": np.ones(20) * 1000,
+    "tradable_shares": np.ones(20) * 1000,
+  })
+
+  # 2. SZ000002: Fails financial growth (150 -> 120 -> 100)
+  df_fin_2 = pl.DataFrame({"ts": dates_fin, "f137": [150.0, 120.0, 100.0]})
+  df_k_2 = df_k_1.clone()
+
+  # 3. SZ000003: Fails large cap (market cap is small)
+  df_fin_3 = df_fin_1.clone()
+  df_k_3 = df_k_1.clone()
+  df_k_3 = df_k_3.with_columns(pl.lit(10.0).alias("total_shares"))
+
+  # 4. SZ000004: Fails MA15 breakdown (stays at 12.0, close is never below MA15)
+  df_fin_4 = df_fin_1.clone()
+  close_4 = np.ones(20) * 12.0
+  df_k_4 = pl.DataFrame({
+    "ts": dates_k,
+    "open": close_4,
+    "high": close_4,
+    "low": close_4,
+    "close": close_4,
+    "volume": np.ones(20) * 1000,
+    "amount": np.ones(20) * 12000,
+    "total_shares": np.ones(20) * 1000,
+    "tradable_shares": np.ones(20) * 1000,
+  })
+
+  # 5. SZ000005: Fails PE filter (PE is too high, cap=40000, net_profit=150, PE=266.67)
+  df_fin_5 = df_fin_1.clone()
+  df_k_5 = df_k_1.clone()
+  df_k_5 = df_k_5.with_columns(pl.lit(5000.0).alias("total_shares"))
+
+  financial_data = {
+    "SZ000001": df_fin_1,
+    "SZ000002": df_fin_2,
+    "SZ000003": df_fin_3,
+    "SZ000004": df_fin_4,
+    "SZ000005": df_fin_5,
+  }
+
+  kline_data = {
+    "SZ000001": df_k_1,
+    "SZ000002": df_k_2,
+    "SZ000003": df_k_3,
+    "SZ000004": df_k_4,
+    "SZ000005": df_k_5,
+  }
+
+  dp = MockSelectorDataProvider(kline_data, financial_data)
+  
+  # Use top 50% percentile
+  selector = GrowthLargeCapMA15Selector(large_cap_percentile=0.5, break_days=3)
+  
+  selected = selector.execute(dp, ["SZ000001", "SZ000002", "SZ000003", "SZ000004", "SZ000005"])
+  
+  assert selected == ["SZ000001"]
+  
+  # Check factors
+  factors = selector.factors
+  assert len(factors) == 1
+  assert factors[0, "symbol"] == "SZ000001"
+  assert factors[0, "net_profit_y3"] == 150.0
+  assert factors[0, "market_cap"] == 8000.0
+  assert factors[0, "pe"] == 8000.0 / 150.0
+  assert factors[0, "close"] == 8.0