pfund 0.0.1.dev11__tar.gz → 0.0.1.dev12__tar.gz

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pfund
-Version: 0.0.1.dev11
+Version: 0.0.1.dev12
 Summary: A Complete Algo-Trading Framework for Machine Learning, enabling trading across TradFi, CeFi and DeFi. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading
 Home-page: https://pfund.ai
 License: Apache-2.0

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/analyzer.py

@@ -1,7 +1,11 @@
+from __future__ import annotations
+
 import os
 from pathlib import Path
 
-from typing import Literal
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfund.types.common_literals import tSUPPORTED_CODE_EDITORS, tSUPPORTED_TEMPLATE_TYPES
 
 from pfund.utils import utils
 from pfund.config_handler import ConfigHandler
@@ -30,7 +34,7 @@ class Analyzer:
         return False
     
     @staticmethod
-    def _derive_template_type(template: str) -> Literal['notebook', 'spreadsheet', 'dashboard']:
+    def _derive_template_type(template: str) -> tSUPPORTED_TEMPLATE_TYPES:
         if '.ipynb' in template:
             template_type = 'notebook'
         elif '.grid' in template:
@@ -56,7 +60,7 @@ class Analyzer:
         else:
             raise FileNotFoundError(f"Template {template} not found in pfund's templates or user's templates")
     
-    def _get_editor_cmd(self, editor: Literal['vscode', 'pycharm']) -> str:
+    def _get_editor_cmd(self, editor: tSUPPORTED_CODE_EDITORS) -> str:
         if editor == 'vscode':
             cmd = 'code'
             if utils.is_command_available(cmd):
@@ -82,7 +86,7 @@ class Analyzer:
         show_results_only: bool=True,
         open_outputs: bool=False,
         outputs_path: str | None=None,
-        editor: Literal['vscode', 'pycharm']='vscode'
+        editor: tSUPPORTED_CODE_EDITORS='vscode'
     ) -> None:
         '''
         Args:
@@ -183,7 +187,8 @@ class Analyzer:
             for process in voila_processes:
                 process.terminate()
                 process.wait()
-        
+    
+    # TODO:
     def run_spreadsheets(
         self, 
         spreadsheets: list[str] | str

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/brokers/ib/broker_ib.py

@@ -5,6 +5,7 @@ from collections import defaultdict
 
 from pfund.adapter import Adapter
 from pfund.config.configuration import Configuration
+from pfund.const.paths import PROJ_CONFIG_PATH
 from pfund.const.commons import SUPPORTED_PRODUCT_TYPES
 from pfund.products import IBProduct
 from pfund.accounts import IBAccount
@@ -19,8 +20,9 @@ from pfund.brokers.ib.ib_api import IBApi
 class IBBroker(LiveBroker):
     def __init__(self, env, **configs):
         super().__init__(env, 'IB', **configs)
-        self.configs = Configuration(self.bkr, 'config')
-        self.adapter = Adapter(self.bkr, self.configs.load_config_section('adapter'))
+        config_path = f'{PROJ_CONFIG_PATH}/{self.bkr.lower()}'
+        self.configs = Configuration(config_path, 'config')
+        self.adapter = Adapter(config_path, self.configs.load_config_section('adapter'))
         self.account = None
         
         # API

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/const/commons.py

@@ -20,4 +20,6 @@ SUPPORTED_TIMEFRAMES = [
 ]
 SUPPORTED_DATA_CHANNELS = ['orderbook', 'tradebook', 'kline']
 SUPPORTED_BACKTEST_MODES = ['vectorized', 'event_driven']
-SUPPORTED_DATA_TOOLS = ['pandas']
+SUPPORTED_DATA_TOOLS = ['pandas']
+SUPPORTED_CODE_EDITORS = ['vscode', 'pycharm']
+SUPPORTED_TEMPLATE_TYPES = ['notebook', 'spreadsheet', 'dashboard']

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/engines/backtest_engine.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+import inspect
 import os
 import time
 import datetime
@@ -13,6 +14,7 @@ if TYPE_CHECKING:
     from pfund.types.core import tStrategy, tModel, tFeature, tIndicator
     
 import pfund as pf
+from pfund.git_controller import GitController
 from pfund.engines.base_engine import BaseEngine
 from pfund.brokers.broker_backtest import BacktestBroker
 from pfund.strategies.strategy_base import BaseStrategy
@@ -23,7 +25,7 @@ from pfund.mixins.backtest import BacktestMixin
 
 
 class BacktestEngine(BaseEngine):
-    def __new__(cls, *, env: str='BACKTEST', data_tool: tSUPPORTED_DATA_TOOLS='pandas', mode: tSUPPORTED_BACKTEST_MODES='vectorized', append_to_strategy_df=False, use_prepared_signals=True, config: ConfigHandler | None=None, **settings):
+    def __new__(cls, *, env: str='BACKTEST', data_tool: tSUPPORTED_DATA_TOOLS='pandas', mode: tSUPPORTED_BACKTEST_MODES='vectorized', append_to_strategy_df=False, use_prepared_signals=True, config: ConfigHandler | None=None, auto_git_commit=False, **settings):
         if not hasattr(cls, 'mode'):
             cls.mode = mode.lower()
         if not hasattr(cls, 'append_to_strategy_df'):
@@ -32,13 +34,18 @@ class BacktestEngine(BaseEngine):
         # instead of recalculating the signals. This will make event-driven backtesting faster but less consistent with live trading
         if not hasattr(cls, 'use_prepared_signals'):
             cls.use_prepared_signals = use_prepared_signals
+        if not hasattr(cls, 'auto_git_commit'):
+            cls.auto_git_commit = auto_git_commit
         return super().__new__(cls, env, data_tool=data_tool, config=config, **settings)
 
-    def __init__(self, *, env: str='BACKTEST', data_tool: tSUPPORTED_DATA_TOOLS='pandas', mode: tSUPPORTED_BACKTEST_MODES='vectorized', append_to_strategy_df=False, use_prepared_signals=True, config: ConfigHandler | None=None, **settings):
-        super().__init__(env, data_tool=data_tool)
+    def __init__(self, *, env: str='BACKTEST', data_tool: tSUPPORTED_DATA_TOOLS='pandas', mode: tSUPPORTED_BACKTEST_MODES='vectorized', append_to_strategy_df=False, use_prepared_signals=True, config: ConfigHandler | None=None, auto_git_commit=False, **settings):
         # avoid re-initialization to implement singleton class correctly
-        # if not hasattr(self, '_initialized'):
-        #     pass
+        if not hasattr(self, '_initialized'):
+            # Get the current frame and then the outer frame (where the engine instance is created)
+            caller_frame = inspect.currentframe().f_back
+            file_path = caller_frame.f_code.co_filename  # Extract the file path from the frame
+            self._git = GitController(os.path.abspath(file_path))
+            super().__init__(env, data_tool=data_tool)
     
     # HACK: since python doesn't support dynamic typing, true return type should be subclass of BacktestMixin and tStrategy
     # write -> BacktestMixin | tStrategy for better intellisense in IDEs
@@ -131,23 +138,35 @@ class BacktestEngine(BaseEngine):
         except:
             self.logger.exception(f"Error writing to {file_path}:")
     
-    def _generate_backtest_iteration(self, strategy: BaseStrategy) -> int:
+    def _generate_backtest_iteration(self, backtest_hash: str) -> int:
         '''Generate backtest iteration number for the same backtest_hash.
         Read the existing backtest.json file to get the iteration number for the same strategy hash
         If the backtest hash is not found, create a new entry with iteration number 1
         else increment the iteration number by 1.
         '''
-        backtest_hash = self._generate_backtest_hash(strategy)
         file_name = 'backtest.json'
         backtest_json = self.read_json(file_name)
         backtest_json[backtest_hash] = backtest_json.get(backtest_hash, 0) + 1
         self._write_json(file_name, backtest_json)
         return backtest_json[backtest_hash]
     
-    def _output_backtest_results(self, strat: str, df, start_time: float, end_time: float):
+    def _commit_strategy(self, strategy: BaseStrategy) -> str | None:
+        engine_name = self.__class__.__name__
+        strat = strategy.name
+        commit_hash: str | None = self._git.commit(strategy._file_path, f'[PFund] {engine_name}: auto-commit strategy "{strat}"')
+        if commit_hash:
+            self.logger.debug(f"Strategy {strat} committed. {commit_hash=}")
+        else:
+            commit_hash = self._git.get_last_n_commit(n=1)[0]
+            self.logger.debug(f"Strategy {strat} has no changes to commit, return the last {commit_hash=}")
+        return commit_hash
+    
+    def _output_backtest_results(self, strat: str, df, start_time: float, end_time: float, commit_hash: str | None):
         strategy = self.get_strategy(strat)
         backtest_id = self._generate_backtest_id()
+        backtest_hash = self._generate_backtest_hash(strategy)
         backtest_name = self._create_backtest_name(strat, backtest_id)
+        backtest_iter = self._generate_backtest_iteration(backtest_hash)
         local_tz = utils.get_local_timezone()
         duration = end_time - start_time
         df_file_path = os.path.join(self.config.backtest_path, f'{backtest_name}.parquet')
@@ -155,7 +174,10 @@ class BacktestEngine(BaseEngine):
             'metadata': {
                 'pfund_version': pf.__version__,
                 'backtest_id': backtest_id,
-                'backtest_iteration': self._generate_backtest_iteration(strategy),
+                'backtest_hash': backtest_hash,
+                'backtest_name': backtest_name,
+                'backtest_iteration': backtest_iter,
+                'commit_hash': commit_hash,
                 'duration': f'{duration:.2f}s' if duration > 1 else f'{duration*1000:.2f}ms',
                 'start_time': datetime.datetime.fromtimestamp(start_time, tz=local_tz).strftime('%Y-%m-%dT%H:%M:%S%z'),
                 'end_time': datetime.datetime.fromtimestamp(end_time, tz=local_tz).strftime('%Y-%m-%dT%H:%M:%S%z'),
@@ -167,7 +189,7 @@ class BacktestEngine(BaseEngine):
         self.data_tool.output_df_to_parquet(df, df_file_path)
         self._write_json(f'{backtest_name}.json', backtest_history)
         return backtest_history
-    
+        
     def run(self):
         for broker in self.brokers.values():
             broker.start()
@@ -181,11 +203,15 @@ class BacktestEngine(BaseEngine):
                     continue
                 if not hasattr(strategy, 'backtest'):
                     raise Exception(f'Strategy {strat} does not have backtest() method, cannot run vectorized backtesting')
+                if self.auto_git_commit and self._git.is_git_repo():
+                    commit_hash = self._commit_strategy(strategy)
+                else:
+                    commit_hash = None
                 start_time = time.time()
                 strategy.backtest()
                 end_time = time.time()
                 df = strategy.get_df()
-                backtest_history: dict = self._output_backtest_results(strat, df, start_time, end_time)
+                backtest_history: dict = self._output_backtest_results(strat, df, start_time, end_time, commit_hash)
                 backtests[strat] = backtest_history
         elif self.mode == 'event_driven':
             for strat, strategy in self.strategy_manager.strategies.items():

pfund-0.0.1.dev12/pfund/git_controller.py

@@ -0,0 +1,60 @@
+import os
+
+from git import Repo, InvalidGitRepositoryError, NoSuchPathError, Commit
+
+
+class GitController:
+    def __init__(self, file_path: str):
+        # find the root directory of the Git repository that contains the given file
+        try:
+            self._repo = Repo(file_path, search_parent_directories=True)
+            self._repo_path = self._repo.git.rev_parse("--show-toplevel")
+        except (InvalidGitRepositoryError, NoSuchPathError):
+            self._repo = self._repo_path = None
+            print(f'{file_path} is not a git repository')
+
+    def is_git_repo(self):
+        return self._repo is not None
+    
+    def get_last_n_commit(self, n=1) -> list[str]:
+        if not self._repo:
+            raise ValueError('No git repository found')        
+        commits: list[Commit] = list(self._repo.iter_commits(paths=self._repo_path, max_count=n))
+        return [commit.hexsha for commit in commits]
+    
+    def commit(self, file_path: str, commit_message: str) -> str | None:
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f'{file_path} not found')
+        if not self._repo:
+            raise ValueError('No git repository found')
+        try:
+            file_relpath = os.path.relpath(file_path, self._repo_path)
+            changed_files = [d.a_path for d in self._repo.index.diff(None)]
+            if file_relpath in self._repo.untracked_files or file_relpath in changed_files:
+                self._repo.index.add([file_relpath])  # Stage the file
+                commit = self._repo.index.commit(commit_message)  # Commit the changes
+                return commit.hexsha  # Return the commit hash
+            else:
+                print(f"No changes detected in {file_path}.")
+                return None
+        except Exception as e:
+            print(f"Error committing {file_path}: {e}")
+            return None
+    
+    # e.g. git checkout <commit_hash> -- strategy._file_path
+    # and git checkout HEAD -- strategy._file_path
+    def checkout_file_from_commit(self, commit_hash, file_path):
+        """Check out a specific file from a specific commit into the working directory."""
+        try:
+            commit = self._repo.commit(commit_hash)  # Get the specific commit
+            file_relpath = os.path.relpath(file_path, self._repo_path)
+            # Access the blob (file content) for the file at the specified commit
+            blob = commit.tree / file_relpath
+            # Write the content of the blob to the file in the working directory
+            with open(file_path, 'wb') as file:
+                blob.stream_data(file)
+            print(f"File {file_path} has been successfully checked out from commit {commit_hash}.")
+        except KeyError:
+            print(f"File {file_path} not found in commit {commit_hash}.")
+        except Exception as e:
+            print(f"Failed to checkout commit {commit_hash}: {e}")

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/models/model_base.py

@@ -2,7 +2,9 @@
 from __future__ import annotations
 
 import os
+import sys
 import importlib
+from pathlib import Path
 from abc import ABC, abstractmethod
 from collections import defaultdict
 
@@ -41,11 +43,42 @@ if TYPE_CHECKING:
 from pfund.const.paths import MODEL_PATH
 from pfund.models.model_meta import MetaModel
 from pfund.products.product_base import BaseProduct
-from pfund.utils.utils import short_path, get_engine_class
+from pfund.utils.utils import short_path, get_engine_class, load_yaml_file
 from pfund.plogging import create_dynamic_logger
 
 
 class BaseModel(ABC, metaclass=MetaModel):
+    
+    _file_path: Path | None = None  # Get the file path where the model was defined
+    config = {}
+    
+    @classmethod
+    def load_config(cls, config: dict | None=None):
+        if config:
+            cls.config = config
+        elif cls._file_path:
+            for file_name in ['config.yml', 'config.yaml']:
+                if config := load_yaml_file(cls._file_path.parent / file_name):
+                    cls.config = config
+                    break
+    
+    def load_params(self, params: dict | None=None):
+        if params:
+            self.params = params
+        elif self._file_path:
+            for file_name in ['params.yml', 'params.yaml']:
+                if params := load_yaml_file(self._file_path.parent / file_name):
+                    self.params = params
+                    break
+    
+    def __new__(cls, *args, **kwargs):
+        if not cls._file_path:
+            module = sys.modules[cls.__module__]
+            if strategy_file_path := getattr(module, '__file__', None):
+                cls._file_path = Path(strategy_file_path)
+                cls.load_config()
+        return super().__new__(cls)
+    
     def __init__(self, ml_model: MachineLearningModel, *args, **kwargs):
         self._args = args
         self._kwargs = kwargs
@@ -73,6 +106,9 @@ class BaseModel(ABC, metaclass=MetaModel):
         self.models = {}
         self.predictions = {}
         self.data = None  # last data
+        
+        self.params = {}
+        self.load_params()
 
     @abstractmethod
     def predict(self, *args, **kwargs) -> pd.DataFrame | torch.Tensor | np.ndarray:
@@ -92,6 +128,8 @@ class BaseModel(ABC, metaclass=MetaModel):
         return {
             'class': self.__class__.__name__,
             'name': self.name,
+            'config': self.config,
+            'params': self.params,
             'ml_model': self.ml_model,
             'datas': [repr(data) for product in self.datas for data in self.datas[product].values()],
             'models': [model.to_dict() for model in self.models.values()],

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/strategies/strategy_base.py

@@ -2,9 +2,11 @@
 from __future__ import annotations
 
 import os
+import sys
 import time
 import importlib
 import datetime
+from pathlib import Path
 from collections import defaultdict, deque
 from abc import ABC
 
@@ -24,11 +26,42 @@ from pfund.zeromq import ZeroMQ
 from pfund.risk_monitor import RiskMonitor
 from pfund.const.commons import SUPPORTED_CRYPTO_EXCHANGES
 from pfund.strategies.strategy_meta import MetaStrategy
-from pfund.utils.utils import convert_to_uppercases, get_engine_class
+from pfund.utils.utils import convert_to_uppercases, get_engine_class, load_yaml_file
 from pfund.plogging import create_dynamic_logger
 
 
 class BaseStrategy(ABC, metaclass=MetaStrategy):
+
+    _file_path: Path | None = None  # Get the file path where the strategy was defined
+    config = {}
+
+    @classmethod
+    def load_config(cls, config: dict | None=None):
+        if config:
+            cls.config = config
+        elif cls._file_path:
+            for file_name in ['config.yml', 'config.yaml']:
+                if config := load_yaml_file(cls._file_path.parent / file_name):
+                    cls.config = config
+                    break
+    
+    def load_params(self, params: dict | None=None):
+        if params:
+            self.params = params
+        elif self._file_path:
+            for file_name in ['params.yml', 'params.yaml']:
+                if params := load_yaml_file(self._file_path.parent / file_name):
+                    self.params = params
+                    break
+    
+    def __new__(cls, *args, **kwargs):
+        if not cls._file_path:
+            module = sys.modules[cls.__module__]
+            if strategy_file_path := getattr(module, '__file__', None):
+                cls._file_path = Path(strategy_file_path)
+                cls.load_config()
+        return super().__new__(cls)
+    
     def __init__(self, *args, **kwargs):
         self._args = args
         self._kwargs = kwargs
@@ -66,6 +99,9 @@ class BaseStrategy(ABC, metaclass=MetaStrategy):
         self.strategies = {}
         self.predictions = {}
         self.data = None  # last data
+        
+        self.params = {}
+        self.load_params()
 
     def __getattr__(self, attr):
         '''gets triggered only when the attribute is not found'''
@@ -96,6 +132,8 @@ class BaseStrategy(ABC, metaclass=MetaStrategy):
         return {
             'class': self.__class__.__name__,
             'name': self.name,
+            'config': self.config,
+            'params': self.params,
             'accounts': [repr(account) for trading_venue in self.accounts for account in self.accounts[trading_venue].values()],
             'datas': [repr(data) for product in self.datas for data in self.datas[product].values()],
             'strategies': [strategy.to_dict() for strategy in self.strategies.values()],

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/templates/notebooks/pfund-overview.ipynb

@@ -2,19 +2,25 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "testing pfund-overview.ipynb\n"
-     ]
-    }
-   ],
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "# Reload all modules (except those excluded by %aimport) every time before executing the Python code typed.\n",
+    "%autoreload 2\n",
+    "\n",
+    "import pfund as pf\n",
+    "import pfolio as po"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "print('testing pfund-overview.ipynb')"
+    "import quantstats"
    ]
   },
   {

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/types/common_literals.py

@@ -22,4 +22,6 @@ tSUPPORTED_TIMEFRAMES = Literal[
 ]
 tSUPPORTED_DATA_CHANNELS = Literal['orderbook', 'tradebook', 'kline']
 tSUPPORTED_BACKTEST_MODES = Literal['vectorized', 'event_driven']
-tSUPPORTED_DATA_TOOLS = Literal['pandas']
+tSUPPORTED_DATA_TOOLS = Literal['pandas']
+tSUPPORTED_CODE_EDITORS = Literal['vscode', 'pycharm']
+tSUPPORTED_TEMPLATE_TYPES = Literal['notebook', 'spreadsheet', 'dashboard']

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pfund/utils/utils.py

@@ -64,9 +64,18 @@ def lowercase(func):
     return wrapper
 
 
-def load_yaml_file(file_path):
-    with open(file_path, 'r') as f:
-        return yaml.safe_load(f)
+def load_yaml_file(file_path) -> dict | list[dict]:
+    if os.path.exists(file_path):
+        with open(file_path, 'r') as f:
+            contents = list(yaml.safe_load_all(f))
+            if not contents:
+                return {}
+            elif len(contents) == 1:
+                return contents[0]
+            else:
+                return contents
+    else:
+        return {}
 
 
 def flatten_dict(d, parent_key='', sep='.'):

{pfund-0.0.1.dev11 → pfund-0.0.1.dev12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pfund"
-version = "0.0.1.dev11"
+version = "0.0.1.dev12"
 description = "A Complete Algo-Trading Framework for Machine Learning, enabling trading across TradFi, CeFi and DeFi. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading"
 license = "Apache-2.0"
 authors = ["Stephen Yau <softwareentrepreneer+pfund@gmail.com>"]
@@ -47,26 +47,27 @@ optional = true
 
 [tool.poetry.group.dev.dependencies]
 # pfeed = { path = "../pfeed", develop = true, extras = ["df", "data", "boost"] }
-# pfolio = { path = "../pfolio", develop = true, extras = ["bayesian", "data", "portfolio", "temporary"] }
+pfolio = { path = "../pfolio", develop = true, extras = ["bayesian", "data", "portfolio", "temporary"] }
 pybit = "^5.6.2"
 ta-lib = "^0.4.28"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^8.0.0"
+pytest-mock = "^3.14.0"
+pytest-cov = "^5.0.0"
+pytest-xdist = "^3.5.0"
+tox = "^4.14.2"
+faker = "^24.4.0"
 pre-commit = "^3.6.1"
 bandit = "^1.7.7"
 grayskull = "^2.5.3"
-pytest-xdist = "^3.5.0"
-faker = "^24.4.0"
-tox = "^4.14.2"
 mypy = "^1.9.0"
 ruff = "^0.3.5"
-pytest-mock = "^3.14.0"
-pytest-cov = "^5.0.0"
 
 [tool.poetry.group.doc.dependencies]
 jupyter-book = "^1.0.0"
 notebook = "^7.1.2"
+sphinxawesome-theme = "^5.1.1"
 
 [build-system]
 requires = ["poetry-core"]