trade-database-manager 0.0.1.dev2__tar.gz → 0.0.1.dev4__tar.gz

{trade_database_manager-0.0.1.dev2/trade_database_manager.egg-info → trade_database_manager-0.0.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.1.dev2
+Version: 0.0.1.dev4
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux

{trade_database_manager-0.0.1.dev2 → trade_database_manager-0.0.1.dev4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trade_database_manager"
-version = "0.0.1.dev2"
+version = "0.0.1.dev4"
 description = "A wrapper of kdb and sql for convenient trade data management."
 readme = "README.md"
 authors = [
@@ -43,3 +43,7 @@ target-version = ['py310']
 [tool.isort]
 profile = "black"
 line_length = 120
+
+[tool.flake8]
+max-line-length = 120
+ignore = "E203,E501,W503"

{trade_database_manager-0.0.1.dev2 → trade_database_manager-0.0.1.dev4}/trade_database_manager/core/sql/sqlmanager.py

@@ -4,7 +4,7 @@ from functools import partial, reduce
 from typing import Any, Literal, Sequence, Type, Union
 
 import pandas as pd
-from sqlalchemy import Float, Index, Integer, MetaData, String, Table, create_engine, inspect, select, sql, text
+from sqlalchemy import REAL, Index, Integer, MetaData, String, Table, create_engine, inspect, select, sql, text
 from sqlalchemy.dialects.postgresql import insert
 
 from ...config import CONFIG
@@ -27,6 +27,12 @@ def _insert_on_conflict_nothing(table, conn, keys, data_iter):
 
 
 class SqlManager:
+    """
+    This class is used to manage SQL operations.
+
+    :ivar sqlalchemy.engine.Engine engine: An instance of the SQLAlchemy Engine class for executing SQL operations.
+    """
+
     def __init__(self):
         self.engine = create_engine(CONFIG["sqlconnstr"])
 
@@ -37,12 +43,20 @@ class SqlManager:
             return conn.execute(sql_executable)
 
     def add_index(self, table_name: str, columns: Union[str, list[str]], unique: bool = True):
+        """
+        Adds an index to a table.
+
+        :param table_name: The name of the table to add the index to.
+        :type table_name: str
+        :param columns: The column(s) to include in the index. It can be a single column name or a list of column names.
+        :type columns: Union[str, list[str]]
+        :param unique: Whether the index should enforce unique values. Defaults to True.
+        :type unique: bool
+        """
         if isinstance(columns, str):
             columns = [columns]
 
         index_name = f"uix_{table_name}_{'_'.join(columns)}"
-        columns_str = ", ".join(columns)
-        unique_str = "UNIQUE" if unique else ""
         table_meta = MetaData()
         table = Table(table_name, table_meta, autoload_with=self.engine)
         columns = [getattr(table.c, colname) for colname in columns if colname in table.c.keys()]
@@ -57,6 +71,22 @@ class SqlManager:
         other_unique_index_columns: Sequence[str] = (),
         other_non_unique_index_columns: Sequence[str] = (),
     ):
+        """
+        Inserts data into a table.
+
+        :param table_name: The name of the table to insert data into.
+        :type table_name: str
+        :param df: The data to insert. It should be a DataFrame where the column names match the table columns.
+        :type df: pd.DataFrame
+        :param upsert: Whether to update the table if the data already exists. Defaults to True.
+        :type upsert: bool
+        :param other_unique_index_columns: Other columns to enforce unique values on. Defaults to an empty sequence.
+        :type other_unique_index_columns: Sequence[str]
+        :param other_non_unique_index_columns: Other columns to add non-unique indexes to. Defaults to an empty sequence.
+        :type other_non_unique_index_columns: Sequence[str]
+        :return: The number of rows inserted.
+        :rtype: int
+        """
         if_exists: Literal["replace", "append"] = "append"
         inspector = inspect(self.engine)
         new_table = not inspector.has_table(table_name)
@@ -78,25 +108,55 @@ class SqlManager:
         return num_rows
 
     def _convert_to_sqlalchemy_type(self, column_type: Type, **kwargs):
-        if isinstance(column_type, str):
-            column_type = type(column_type)
-        if column_type == str:
+        if isinstance(column_type, type):
+            column_type = column_type.__name__
+        if column_type == "str":
             return String(**kwargs)
-        if column_type == int:
+        if column_type == "int":
             return Integer()
-        if column_type == float:
-            return Float()
+        if column_type == "float":
+            return REAL()
         raise ValueError(f"Unsupported column type {column_type}")
 
     def insert_column(self, table_name: str, column_name: str, column_type: Union[str, Type], type_kwargs: dict = None):
+        """
+        Inserts a new column into a table.
+
+        :param table_name: The name of the table to insert the column into.
+        :type table_name: str
+        :param column_name: The name of the new column.
+        :type column_name: str
+        :param column_type: The data type of the new column. It can be a string or a Python type.
+        :type column_type: Union[str, Type]
+        :param type_kwargs: Additional keyword arguments for the data type. Defaults to None.
+        :type type_kwargs: dict, optional
+        """
         type_kwargs = type_kwargs or {}
         sql_code = f"ALTER TABLE {table_name} ADD COLUMN {column_name} {str(self._convert_to_sqlalchemy_type(column_type, **type_kwargs))}"
         self._execute(sql_code)
 
     def delete_column(self, table_name: str, column_name: str):
+        """
+        Deletes a column from a table.
+
+        :param table_name: The name of the table to delete the column from.
+        :type table_name: str
+        :param column_name: The name of the column to delete.
+        :type column_name: str
+        """
         self._execute(f"ALTER TABLE {table_name} DROP COLUMN {column_name}")
 
     def rename_column(self, table_name: str, old_column_name: str, new_column_name: str):
+        """
+        Renames a column in a table.
+
+        :param table_name: The name of the table containing the column to rename.
+        :type table_name: str
+        :param old_column_name: The current name of the column.
+        :type old_column_name: str
+        :param new_column_name: The new name for the column.
+        :type new_column_name: str
+        """
         # check if any index is referring to the column
         sql_code = f"SELECT indexname, indexdef FROM pg_indexes WHERE indexdef LIKE '%%(%%{old_column_name}%%)%%' and tablename = '{table_name}'"
         index_refering_column = dict(self.engine.execute(sql_code).fetchall())
@@ -154,6 +214,19 @@ class SqlManager:
         return pd.DataFrame(res.fetchall(), columns=res.keys())
 
     def read_data(self, table_name: str, query_fields: QUERYFIELD_TYPE = "*", filter_fields=None):
+        """
+        Reads data from a table.
+
+        :param table_name: The name of the table to read data from.
+        :type table_name: str
+        :param query_fields: The fields to query. By default, it queries all fields. Defaults to "*".
+        :type query_fields: QUERYFIELD_TYPE, optional
+        :param filter_fields: Additional fields to filter by. The keys are the field names and the values are the filter values. Defaults to None.
+        :type filter_fields: dict, optional
+        :return: A DataFrame containing the queried data.
+        :rtype: pd.DataFrame
+        """
+
         meta = MetaData()
         table = Table(table_name, meta, autoload_with=self.engine)
 
@@ -177,6 +250,7 @@ class SqlManager:
 
         # cannot use pandas.read_sql here as it discards timezone info
         res = self._execute(stmt)
+        pd.read_sql_query()
         return pd.DataFrame(res.fetchall(), columns=res.keys())
 
     def read_data_across_tables(
@@ -186,6 +260,20 @@ class SqlManager:
         query_fields: QUERYFIELD_TYPE = "*",
         filter_fields: FILTERFIELD_TYPE = None,
     ):
+        """
+        Reads data from multiple tables.
+
+        :param table_names: The names of the tables to read data from.
+        :type table_names: Sequence[str]
+        :param joined_columns: The columns to join the tables on.
+        :type joined_columns: Sequence[str]
+        :param query_fields: The fields to query. By default, it queries all fields. Defaults to "*".
+        :type query_fields: QUERYFIELD_TYPE, optional
+        :param filter_fields: Additional fields to filter by. The keys are the field names and the values are the filter values. Defaults to None.
+        :type filter_fields: FILTERFIELD_TYPE, optional
+        :return: A DataFrame containing the queried data.
+        :rtype: pd.DataFrame
+        """
         meta = MetaData()
         tables = {table_name: Table(table_name, meta, autoload_with=self.engine) for table_name in table_names}
 

{trade_database_manager-0.0.1.dev2 → trade_database_manager-0.0.1.dev4}/trade_database_manager/manager/__init__.py

@@ -3,3 +3,7 @@
 # @Author  : YQ Tsui
 # @File    : __init__.py
 # @Purpose :
+
+from .metadata_sql import MetadataSql
+
+__all__ = ("MetadataSql",)

{trade_database_manager-0.0.1.dev2 → trade_database_manager-0.0.1.dev4}/trade_database_manager/manager/metadata_sql.py

@@ -5,7 +5,7 @@
 # @Purpose : Instrument metadata stored in SQL database
 
 from collections.abc import Container
-from typing import Union
+from typing import Union, cast
 
 import pandas as pd
 
@@ -14,6 +14,7 @@ from .typedefs import EXCHANGE_LITERALS, INST_TYPE_LITERALS, Opt_T_SeqT, T_DictT
 
 COMMON_METADATA_COLUMNS = [
     "name",
+    "trading_code",
     "inst_type",
     "currency",
     "timezone",
@@ -25,15 +26,33 @@ COMMON_METADATA_COLUMNS = [
     "delisted_date",
 ]
 TYPE_METADATA_COLUMNS = {
-    "STK": ["sector", "industry", "country", "state", "board_type"],
+    "STK": ["sector", "industry", "country", "state", "board_type", "issue_price"],
 }
 
 
 class MetadataSql:
-    def __init__(self):
-        self.manager = SqlManager()
+    """
+    This class is used to manage instrument metadata stored in an SQL database.
+
+    This is a singleton class. Just call MetadataSql() to get the instance.
+    """
+
+    _instance = None
+    _manager = None
+
+    def __new__(cls):
+        if not isinstance(cls._instance, cls):
+            cls._instance = super(MetadataSql, cls).__new__(cls)
+            cls._manager = SqlManager()
+        return cls._instance
 
     def update_instrument_metadata(self, data: Union[pd.DataFrame, list[dict], dict]):
+        """
+        Updates the instrument metadata in the database.
+
+        :param data: The data to be updated. It can be a DataFrame, a list of dictionaries, or a single dictionary.
+        :type data: Union[pd.DataFrame, list[dict], dict]
+        """
         type_specific_columns = set(data.columns) - set(COMMON_METADATA_COLUMNS)
         if "inst_type" not in data.columns and bool(type_specific_columns):
             raise ValueError(
@@ -43,15 +62,23 @@ class MetadataSql:
             data = [data]
         if isinstance(data, list):
             data = pd.DataFrame(data)
-        data.set_index(["ticker", "exchange"], inplace=True)
+        if "ticker" in data.columns and "exchange" in data.columns:
+            data.set_index(["ticker", "exchange"], inplace=True)
+        else:
+            assert set(data.index.names) == {"ticker", "exchange"}, "Index names must be 'ticker' and 'exchange'."
         data_common = data[data.columns.intersection(COMMON_METADATA_COLUMNS)]
 
-        self.manager.insert("instruments", data_common, upsert=True)
+        self._manager.insert("instruments", data_common, upsert=True)
         if "inst_type" in data.columns:
             for inst_type, columns in TYPE_METADATA_COLUMNS.items():
                 data_type_df = data.loc[data.inst_type == inst_type, data.columns.intersection(columns)]
                 if not data_type_df.empty:
-                    self.manager.insert(f"instruments_{inst_type.lower()}", data_type_df, upsert=True)
+                    self._manager.insert(f"instruments_{inst_type.lower()}", data_type_df, upsert=True)
+
+    def _convert_datetime_columns(self, data: pd.DataFrame):
+        for col in ["listed_date", "delisted_date"]:
+            if col in data.columns:
+                data[col] = data[col].apply(pd.to_datetime)
 
     def read_metadata(
         self,
@@ -60,11 +87,25 @@ class MetadataSql:
         query_fields="*",
         filter_fields=None,
     ) -> T_DictT[pd.DataFrame]:
+        """
+        Reads metadata from the database based on the provided filters.
+
+        :param ticker: The ticker(s) to filter by. It can be a single ticker or a sequence of tickers. Defaults to None.
+        :type ticker: Opt_T_SeqT[str], optional
+        :param exchange: The exchange(s) to filter by. It can be a single exchange or a sequence of exchanges. Defaults to None.
+        :type exchange: Opt_T_SeqT[EXCHANGE_LITERALS], optional
+        :param query_fields: The fields to query. By default, it queries all fields. Defaults to "*".
+        :type query_fields: str, optional
+        :param filter_fields: Additional fields to filter by. The keys are the field names and the values are the filter values. Defaults to None.
+        :type filter_fields: dict, optional
+        :return: A dictionary of DataFrames containing the queried metadata.
+        :rtype: T_DictT[pd.DataFrame]
+        """
         filter_fields = filter_fields or {}
         if ticker is not None:
             filter_fields["ticker"] = ticker
         if exchange is not None:
-            if not isinstance(exchange, str) and isinstance(exchange, Container):
+            if not isinstance(exchange, str) and isinstance(exchange, Container) and ticker is not None:
                 assert len(exchange) == len(ticker), "Exchange must be a single value or the same length as ticker."
             filter_fields["exchange"] = exchange
         query_fields_common = (
@@ -82,11 +123,15 @@ class MetadataSql:
         )
         if not (query_fields == "*" or "inst_type" in query_fields):
             query_fields_common.append("inst_type")
-        common_df = self.manager.read_data("instruments", query_fields=query_fields_common, filter_fields=filter_fields)
+        common_df = self._manager.read_data(
+            "instruments", query_fields=query_fields_common, filter_fields=filter_fields
+        )
+        self._convert_datetime_columns(common_df)
         if common_df.empty:
             return {}
         res = {}
         for inst_type, common_df_by_type in common_df.groupby("inst_type"):
+            inst_type = cast(INST_TYPE_LITERALS, inst_type)
             if all_fields_common:
                 res[inst_type] = common_df_by_type
                 continue
@@ -98,7 +143,7 @@ class MetadataSql:
             filter_fields_type = {k: v for k, v in filter_fields.items() if k in TYPE_METADATA_COLUMNS[inst_type]}
             filter_fields_type["ticker"] = common_df_by_type["ticker"].to_list()
             filter_fields_type["exchange"] = common_df_by_type["exchange"].to_list()
-            type_df = self.manager.read_data(
+            type_df = self._manager.read_data(
                 f"instruments_{inst_type.lower()}", query_fields=query_fields_type, filter_fields=filter_fields_type
             )
             type_df = common_df_by_type.merge(type_df, on=["ticker", "exchange"], how="inner")
@@ -113,12 +158,28 @@ class MetadataSql:
         query_fields="*",
         filter_fields=None,
     ) -> pd.DataFrame:
+        """
+        Reads metadata for a specific instrument type from the database based on the provided filters.
+
+        :param inst_type: The instrument type to filter by.
+        :type inst_type: INST_TYPE_LITERALS
+        :param ticker: The ticker(s) to filter by. It can be a single ticker or a sequence of tickers. Defaults to None.
+        :type ticker: Opt_T_SeqT[str], optional
+        :param exchange: The exchange(s) to filter by. It can be a single exchange or a sequence of exchanges. Defaults to None.
+        :type exchange: Opt_T_SeqT[EXCHANGE_LITERALS], optional
+        :param query_fields: The fields to query. By default, it queries all fields. Defaults to "*".
+        :type query_fields: str, optional
+        :param filter_fields: Additional fields to filter by. The keys are the field names and the values are the filter values. Defaults to None.
+        :type filter_fields: dict, optional
+        :return: A DataFrame containing the queried metadata for the specified instrument type.
+        :rtype: pd.DataFrame
+        """
         filter_fields = filter_fields or {}
         filter_fields["inst_type"] = inst_type
         if ticker is not None:
             filter_fields["ticker"] = ticker
         if exchange is not None:
-            if not isinstance(exchange, str) and isinstance(exchange, Container):
+            if not isinstance(exchange, str) and isinstance(exchange, Container) and ticker is not None:
                 assert len(exchange) == len(ticker), "Exchange must be a single value or the same length as ticker."
             filter_fields["exchange"] = exchange
 
@@ -144,12 +205,16 @@ class MetadataSql:
             "instruments": filter_fields_common,
             f"instruments_{inst_type.lower()}": filter_fields_type,
         }
-        df = self.manager.read_data_across_tables(
+        df = self._manager.read_data_across_tables(
             ["instruments", f"instruments_{inst_type.lower()}"],
             joined_columns=["ticker", "exchange"],
             query_fields=query_fields_cross,
             filter_fields=filter_fields_cross,
         )
+
         if isinstance(df.columns, pd.Index):
-            return df.loc[:, ~df.columns.duplicated()].set_index(["ticker", "exchange"])
-        return df.set_index(["ticker", "exchange"])
+            df = df.loc[:, ~df.columns.duplicated()].set_index(["ticker", "exchange"])
+        else:
+            df = df.set_index(["ticker", "exchange"])
+        self._convert_datetime_columns(df)
+        return df

{trade_database_manager-0.0.1.dev2 → trade_database_manager-0.0.1.dev4/trade_database_manager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.1.dev2
+Version: 0.0.1.dev4
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux