investing-algorithm-framework 8.7.0__tar.gz → 8.7.2__tar.gz

{investing_algorithm_framework-8.7.0 → investing_algorithm_framework-8.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: investing-algorithm-framework
-Version: 8.7.0
+Version: 8.7.2
 Summary: A framework for creating trading bots
 Author: MDUYN
 Requires-Python: >=3.10,<4.0

{investing_algorithm_framework-8.7.0 → investing_algorithm_framework-8.7.2}/investing_algorithm_framework/__init__.py

@@ -17,6 +17,7 @@ from .domain import ApiException, combine_backtests, PositionSize, \
     Position, TimeFrame, INDEX_DATETIME, MarketCredential, TakeProfitRule, \
     PortfolioConfiguration, RESOURCE_DIRECTORY, AWS_LAMBDA_LOGGING_CONFIG, \
     Trade, APP_MODE, AppMode, DATETIME_FORMAT, load_backtests_from_directory, \
+    iter_backtests_from_directory, \
     BacktestDateRange, convert_polars_to_pandas, BacktestRun, \
     DEFAULT_LOGGING_CONFIG, DataType, DataProvider, StopLossRule, \
     ScalingRule, TradingCost, \
@@ -24,7 +25,7 @@ from .domain import ApiException, combine_backtests, PositionSize, \
     APPLICATION_DIRECTORY, DataSource, OrderExecutor, PortfolioProvider, \
     SnapshotInterval, AWS_S3_STATE_BUCKET_NAME, BacktestEvaluationFocus, \
     save_backtests_to_directory, BacktestMetrics, DATA_DIRECTORY, \
-    retag_backtests, \
+    retag_backtests, migrate_backtests, \
     Blotter, DefaultBlotter, SimulationBlotter, Transaction, \
     SlippageModel, NoSlippage, PercentageSlippage, FixedSlippage, \
     VolumeImpactSlippage, \
@@ -71,7 +72,8 @@ from .services import get_annual_volatility, get_sortino_ratio, \
     get_current_average_trade_loss, get_negative_trades, \
     get_positive_trades, get_number_of_trades, get_current_win_rate, \
     get_current_win_loss_ratio, create_backtest_metrics_for_backtest, \
-    recalculate_backtests, TradeTakeProfitService, TradeStopLossService, \
+    recalculate_backtests, recalculate_backtests_in_directory, \
+    TradeTakeProfitService, TradeStopLossService, \
     get_cv_consistency, get_normalized_stability, \
     get_consistency_score, get_stability_score
 
@@ -220,11 +222,14 @@ __all__ = [
     "get_number_of_trades",
     "BacktestRun",
     "load_backtests_from_directory",
+    "iter_backtests_from_directory",
     "save_backtests_to_directory",
     "retag_backtests",
+    "migrate_backtests",
     "DataError",
     "create_backtest_metrics_for_backtest",
     "recalculate_backtests",
+    "recalculate_backtests_in_directory",
     "TakeProfitRule",
     "StopLossRule",
     "ScalingRule",

{investing_algorithm_framework-8.7.0 → investing_algorithm_framework-8.7.2}/investing_algorithm_framework/domain/__init__.py

@@ -44,6 +44,7 @@ from .backtesting import BacktestRun, BacktestSummaryMetrics, \
     BacktestDateRange, Backtest, BacktestMetrics, combine_backtests, \
     BacktestPermutationTest, BacktestEvaluationFocus, \
     generate_backtest_summary_metrics, load_backtests_from_directory, \
+    iter_backtests_from_directory, \
     save_backtests_to_directory, retag_backtests, migrate_backtests, \
     BacktestIndex, save_bundle, open_bundle, BUNDLE_EXT, \
     BUNDLE_FORMAT_VERSION
@@ -174,6 +175,7 @@ __all__ = [
     'ScalingRule',
     'TradingCost',
     "load_backtests_from_directory",
+    "iter_backtests_from_directory",
     "save_backtests_to_directory",
     "retag_backtests",
     "migrate_backtests",

{investing_algorithm_framework-8.7.0 → investing_algorithm_framework-8.7.2}/investing_algorithm_framework/domain/backtesting/__init__.py

@@ -9,6 +9,7 @@ from .combine_backtests import combine_backtests, \
     generate_backtest_summary_metrics
 from .backtest_utils import (
     load_backtests_from_directory,
+    iter_backtests_from_directory,
     save_backtests_to_directory,
     retag_backtests,
     migrate_backtests,
@@ -33,6 +34,7 @@ __all__ = [
     "combine_backtests",
     "generate_backtest_summary_metrics",
     "load_backtests_from_directory",
+    "iter_backtests_from_directory",
     "save_backtests_to_directory",
     "retag_backtests",
     "migrate_backtests",

{investing_algorithm_framework-8.7.0 → investing_algorithm_framework-8.7.2}/investing_algorithm_framework/domain/backtesting/backtest_utils.py

@@ -1,6 +1,7 @@
 import json
 import os
-from concurrent.futures import ProcessPoolExecutor, as_completed
+from concurrent.futures import ProcessPoolExecutor, as_completed, \
+    wait, FIRST_COMPLETED
 from logging import getLogger
 from pathlib import Path
 from random import Random
@@ -409,22 +410,47 @@ def load_backtests_from_directory(
     else:
         # Process pool only handles bundle loads efficiently; loading
         # legacy directories from workers also works but the win is
-        # smaller. We still parallelise both for simplicity.
-        with ProcessPoolExecutor(max_workers=n_workers) as ex:
-            futures = {
-                ex.submit(_load_one_dispatch, it): it for it in sources
-            }
-            for fut in as_completed(futures):
-                src = futures[fut]
+        # smaller. We still parallelise both for simplicity. Submission
+        # is bounded to ``n_workers`` in flight so the executor's
+        # feeder thread does not buffer the entire source list, and
+        # workers are recycled (Python 3.11+) so RSS stays bounded.
+        import sys
+        pool_kwargs = {"max_workers": n_workers}
+        if sys.version_info >= (3, 11):
+            pool_kwargs["max_tasks_per_child"] = 16
+
+        sources_iter = iter(sources)
+        inflight = {}
+
+        with ProcessPoolExecutor(**pool_kwargs) as ex:
+            def _submit_next():
                 try:
-                    backtests.append(fut.result())
-                except Exception as e:
-                    logger.error(
-                        f"Failed to load backtest from {src[0]}: {e}"
-                    )
-                finally:
-                    if pbar is not None:
-                        pbar.update(1)
+                    it = next(sources_iter)
+                except StopIteration:
+                    return False
+                inflight[ex.submit(_load_one_dispatch, it)] = it
+                return True
+
+            for _ in range(n_workers):
+                if not _submit_next():
+                    break
+
+            while inflight:
+                done_set, _unused = wait(
+                    inflight.keys(), return_when=FIRST_COMPLETED
+                )
+                for fut in done_set:
+                    src = inflight.pop(fut)
+                    try:
+                        backtests.append(fut.result())
+                    except Exception as e:
+                        logger.error(
+                            f"Failed to load backtest from {src[0]}: {e}"
+                        )
+                    finally:
+                        if pbar is not None:
+                            pbar.update(1)
+                    _submit_next()
 
     if pbar is not None:
         pbar.close()
@@ -438,6 +464,87 @@ def load_backtests_from_directory(
     return backtests
 
 
+def iter_backtests_from_directory(
+    directory_path: Union[str, Path],
+    filter_function: Callable[[Backtest], bool] = None,
+    number_of_backtests_to_load: int = None,
+    show_progress: bool = False,
+):
+    """Yield :class:`Backtest` objects one at a time from *directory_path*.
+
+    Memory-stable alternative to :func:`load_backtests_from_directory`:
+    each backtest is loaded only when the caller asks for the next
+    item, so peak memory equals one backtest at a time. Use this when
+    a directory contains so many large backtests that the full list
+    would not fit in RAM.
+
+    Example:
+        >>> for bt in iter_backtests_from_directory(path):
+        ...     process(bt)
+        ...     # bt is dropped here; its memory is reclaimed before
+        ...     # the next one is read from disk.
+
+    Args:
+        directory_path: Source directory.
+        filter_function: Optional predicate; backtests for which it
+            returns False are skipped.
+        number_of_backtests_to_load: Optional cap on the number of
+            backtests yielded.
+        show_progress: Display a tqdm progress bar.
+    """
+    directory_path = str(directory_path)
+    if not os.path.exists(directory_path):
+        logger.warning(
+            f"Directory {directory_path} does not exist. "
+            "No backtests loaded."
+        )
+        return
+
+    sources: List[tuple] = []
+    for entry in sorted(os.listdir(directory_path)):
+        if entry == "checkpoints.json" or entry.endswith(".py"):
+            continue
+        if entry == "index.parquet" or entry == "ohlcv":
+            continue
+        full = os.path.join(directory_path, entry)
+        if os.path.isfile(full):
+            if entry.endswith(BUNDLE_EXT) or is_bundle_file(full):
+                sources.append((full, "bundle"))
+        elif os.path.isdir(full):
+            sources.append((full, "directory"))
+
+    if number_of_backtests_to_load is not None:
+        sources = sources[: max(0, int(number_of_backtests_to_load))]
+
+    pbar = tqdm(
+        total=len(sources),
+        desc="Loading backtests",
+        disable=not show_progress,
+    )
+    try:
+        for item in sources:
+            try:
+                bt = _load_one_dispatch(item)
+            except Exception as e:
+                logger.error(
+                    f"Failed to load backtest from {item[0]}: {e}"
+                )
+                pbar.update(1)
+                continue
+            try:
+                if filter_function is not None and not filter_function(bt):
+                    pbar.update(1)
+                    continue
+            except Exception as fe:  # pragma: no cover - defensive
+                logger.error(f"Error in filter_function: {fe}")
+                pbar.update(1)
+                continue
+            yield bt
+            pbar.update(1)
+    finally:
+        pbar.close()
+
+
 # ---------------------------------------------------------------------------
 # Index helpers (#487 step 4)
 # ---------------------------------------------------------------------------
@@ -570,13 +677,14 @@ class BacktestIndex:
 def _migrate_one(args):
     """Worker entry point: open *src* (legacy dir or bundle), write
     *dst* as a bundle, optionally delete *src*, return the
-    destination path.
+    destination path together with a flat index row.
 
     Doing load+save (and optionally delete) in one worker call keeps
     each backtest's data in a single process — avoiding the cost of
     pickling fully-decoded Backtest objects back to the parent
-    process. This roughly halves peak memory usage for large
-    migrations and is faster end-to-end.
+    process. We also build the index row here, while the backtest is
+    still in memory, so the parent never has to re-open the migrated
+    bundles just to build ``index.parquet``.
     """
     src, dst, include_ohlcv, ohlcv_store, delete_source = args
     bt = _open_bundle(src) if is_bundle_file(src) else Backtest.open(src)
@@ -585,6 +693,11 @@ def _migrate_one(args):
         include_ohlcv=include_ohlcv,
         ohlcv_store=ohlcv_store,
     ))
+    rel = os.path.basename(out)
+    row = _backtest_to_index_row(bt, bundle_path=rel)
+    # Drop the heavy backtest before returning so the worker process's
+    # RSS can be reclaimed before it picks up the next task.
+    del bt
     if delete_source and os.path.abspath(src) != os.path.abspath(out):
         import shutil
         if os.path.isdir(src):
@@ -594,7 +707,7 @@ def _migrate_one(args):
                 os.remove(src)
             except OSError:
                 pass
-    return out
+    return out, row
 
 
 def migrate_backtests(
@@ -681,33 +794,74 @@ def migrate_backtests(
     n = len(plan)
     resolved_workers = min(_resolve_workers(workers), n)
 
-    iterator: object
-    if resolved_workers > 1:
-        with ProcessPoolExecutor(max_workers=resolved_workers) as ex:
-            results = ex.map(_migrate_one, plan)
-            iterator = tqdm(
-                results,
-                total=n,
-                desc="Migrating backtests",
-                disable=not show_progress,
-            )
-            for _ in iterator:
-                pass
-    else:
-        for args in tqdm(
-            plan,
-            total=n,
-            desc="Migrating backtests",
-            disable=not show_progress,
-        ):
-            _migrate_one(args)
-
-    if write_index:
-        # Re-open the freshly written bundles (cheap header reads only
-        # for the index) to build the parquet manifest.
-        migrated = load_backtests_from_directory(
-            dst_dir, workers=workers, show_progress=False,
+    rows: List[dict] = []
+    pbar = tqdm(
+        total=n,
+        desc="Migrating backtests",
+        disable=not show_progress,
+    )
+    try:
+        if resolved_workers > 1:
+            # Bound in-flight tasks to ``resolved_workers`` so we don't
+            # buffer the full plan inside the executor's feeder, and
+            # consume results as they finish to keep memory flat.
+            # Recycle workers (3.11+) so RSS stays bounded across long
+            # migrations.
+            import sys
+            pool_kwargs = {"max_workers": resolved_workers}
+            if sys.version_info >= (3, 11):
+                pool_kwargs["max_tasks_per_child"] = 16
+            plan_iter = iter(plan)
+            with ProcessPoolExecutor(**pool_kwargs) as ex:
+                inflight = {}
+                for _ in range(resolved_workers):
+                    try:
+                        args = next(plan_iter)
+                    except StopIteration:
+                        break
+                    inflight[ex.submit(_migrate_one, args)] = args
+
+                while inflight:
+                    done_set, _unused = wait(
+                        inflight.keys(), return_when=FIRST_COMPLETED
+                    )
+                    for fut in done_set:
+                        args = inflight.pop(fut)
+                        try:
+                            _out, row = fut.result()
+                            if write_index:
+                                rows.append(row)
+                        except Exception as e:
+                            logger.error(
+                                f"Failed to migrate {args[0]}: {e}"
+                            )
+                        finally:
+                            pbar.update(1)
+                        try:
+                            nxt = next(plan_iter)
+                        except StopIteration:
+                            continue
+                        inflight[ex.submit(_migrate_one, nxt)] = nxt
+        else:
+            for args in plan:
+                try:
+                    _out, row = _migrate_one(args)
+                    if write_index:
+                        rows.append(row)
+                except Exception as e:
+                    logger.error(f"Failed to migrate {args[0]}: {e}")
+                finally:
+                    pbar.update(1)
+    finally:
+        pbar.close()
+
+    if write_index and rows:
+        import pandas as pd  # local import keeps top of module light
+        df = pd.DataFrame(rows)
+        df.to_parquet(
+            Path(dst_dir) / "index.parquet",
+            index=False,
+            compression="zstd",
         )
-        _write_index(dst_dir, migrated)
 
     return n

{investing_algorithm_framework-8.7.0 → investing_algorithm_framework-8.7.2}/investing_algorithm_framework/services/__init__.py

@@ -38,7 +38,8 @@ from .metrics import get_annual_volatility, get_mean_daily_return, \
     get_current_win_rate, get_current_average_trade_return, \
     get_current_average_trade_loss, get_current_average_trade_duration, \
     get_current_average_trade_gain, create_backtest_metrics_for_backtest, \
-    recalculate_backtests, get_cv_consistency, get_normalized_stability, \
+    recalculate_backtests, recalculate_backtests_in_directory, \
+    get_cv_consistency, get_normalized_stability, \
     get_consistency_score, get_stability_score
 
 __all__ = [
@@ -133,6 +134,7 @@ __all__ = [
     "get_current_average_trade_return",
     "create_backtest_metrics_for_backtest",
     "recalculate_backtests",
+    "recalculate_backtests_in_directory",
     "TradeStopLossService",
     "TradeTakeProfitService",
     "get_mean_yearly_return",

{investing_algorithm_framework-8.7.0 → investing_algorithm_framework-8.7.2}/investing_algorithm_framework/services/metrics/__init__.py

@@ -28,7 +28,8 @@ from .win_rate import get_win_rate, get_win_loss_ratio, get_current_win_rate, \
     get_current_win_loss_ratio
 from .calmar_ratio import get_calmar_ratio
 from .generate import create_backtest_metrics, \
-    create_backtest_metrics_for_backtest, recalculate_backtests
+    create_backtest_metrics_for_backtest, recalculate_backtests, \
+    recalculate_backtests_in_directory
 from .risk_free_rate import get_risk_free_rate_us
 from .trades import get_negative_trades, get_positive_trades, \
     get_number_of_trades, get_number_of_closed_trades, \