trade-database-manager 0.0.3.dev1__tar.gz → 0.0.6.dev0__tar.gz

{trade_database_manager-0.0.3.dev1/trade_database_manager.egg-info → trade_database_manager-0.0.6.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.3.dev1
+Version: 0.0.6.dev0
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6.dev0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trade_database_manager"
-version = "0.0.3.dev1"
+version = "0.0.6.dev0"
 description = "A wrapper of kdb and sql for convenient trade data management."
 readme = "README.md"
 authors = [
@@ -30,7 +30,7 @@ include = ["trade_database_manager*"]
 const-rgx='[a-z_][a-z0-9_]{2,30}$'
 
 [tool.pylint.main]
-disable="C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R1702,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136"
+disable="C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R1702,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R0917,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136"
 ignore-paths="\\.ipynb_checkpoints/*"
 
 [tool.pylint.format]
@@ -43,7 +43,3 @@ target-version = ['py310']
 [tool.isort]
 profile = "black"
 line_length = 120
-
-[tool.flake8]
-max-line-length = 120
-ignore = "E203,E501,W503"

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6.dev0}/trade_database_manager/core/kdb/__init__.py

@@ -2,4 +2,4 @@
 # @Time    : 2024/4/15 20:28
 # @Author  : YQ Tsui
 # @File    : __init__.py
-# @Purpose :
+# @Purpose :

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6.dev0}/trade_database_manager/core/kdb/kdbmanager.py

@@ -17,7 +17,6 @@ class KdbManager:
     def instance(cls):
         if cls._instance is None:
             cls._instance = cls()
-            cls._instance.__init__()
         return cls._instance
 
     def __init__(self):
@@ -47,7 +46,7 @@ class KdbManager:
         :type path: str
         """
         with pykx.QConnection(self.host, self.port, username=self.username, password=self.password) as conn:
-            conn(f".path.mkdir \"{path}\"")
+            conn(f'.path.mkdir "{path}"')
 
     def write(self, table_name: str, data: pd.DataFrame, path: str = "", splayed: bool = False):
         """
@@ -69,7 +68,9 @@ class KdbManager:
         with pykx.QConnection(self.host, self.port, username=self.username, password=self.password) as conn:
             conn(f"{{`:{real_path}/ set x}}", data)
 
-    def write_partitioned(self, table_name: str, data: pd.DataFrame, path: str = "", partition_func=None, key_column=None):
+    def write_partitioned(
+        self, table_name: str, data: pd.DataFrame, path: str = "", partition_func=None, key_column=None
+    ):
         """
         Writes data to the kdb database with partitioning.
 
@@ -89,16 +90,31 @@ class KdbManager:
         assert "datetime" in data.columns, "datetime column not found"
         data.sort_values(by="datetime", inplace=True)
         if key_column is not None and key_column in data.columns:
-            with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
+            with pykx.QConnection(
+                host=self.host, port=self.port, username=self.username, password=self.password
+            ) as conn:
                 for bucket, df in data.groupby(partition_func(data["datetime"])):
-                    conn(f'{{`{table_name} set x; .partable.createOrAppend[`:{path};{bucket};`{key_column};`{table_name}]}}', df.reset_index(drop=True))
+                    conn(
+                        f"{{`{table_name} set x; .partable.createOrAppend[`:{path};{bucket};`{key_column};`{table_name}]}}",
+                        df.reset_index(drop=True),
+                    )
         else:
-            with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
+            with pykx.QConnection(
+                host=self.host, port=self.port, username=self.username, password=self.password
+            ) as conn:
                 for bucket, df in data.groupby(by=partition_func(data["datetime"])):
-                    conn(f'{{`{table_name} set x;.Q.dpt[`:{path};{bucket};`{table_name}]}}', df.reset_index(drop=True))
-
-    def read_partitioned(self, table_name: str, path: str = "", fields=None, start_time=None, end_time=None, partition_func=None,
-                         other_conditions=None):
+                    conn(f"{{`{table_name} set x;.Q.dpt[`:{path};{bucket};`{table_name}]}}", df.reset_index(drop=True))
+
+    def read_partitioned(
+        self,
+        table_name: str,
+        path: str = "",
+        fields=None,
+        start_time=None,
+        end_time=None,
+        partition_func=None,
+        other_conditions=None,
+    ):
         """
         Reads data from the kdb database with partitioning.
 
@@ -120,31 +136,39 @@ class KdbManager:
         end_time_str = end_time.strftime(time_format) if end_time is not None else None
         if isinstance(fields, (bytes, str)):
             fields = [fields]
-        select_clause = f"select from {table_name}" if fields is None else f"select {','.join(fields)} from {table_name}"
+        select_clause = (
+            f"select from {table_name}" if fields is None else f"select {','.join(fields)} from {table_name}"
+        )
         where_cond = ""
         if start_time_str is not None and end_time_str is not None:
             where_cond += f"datetime within ({start_time_str};{end_time_str})"
             if partition_func is not None:
-                where_cond = f"int in {' '.join(str(x) for x in range(partition_func(start_time), partition_func(end_time) + 1))}" + ',' + where_cond
+                where_cond = (
+                    f"int in {' '.join(str(x) for x in range(partition_func(start_time), partition_func(end_time) + 1))}"
+                    + ","
+                    + where_cond
+                )
         elif start_time_str is not None:
             where_cond += f"datetime>={start_time_str}"
             if partition_func is not None:
-                where_cond = f"int>={partition_func(start_time)}" + ',' + where_cond
+                where_cond = f"int>={partition_func(start_time)}" + "," + where_cond
         elif end_time_str is not None:
             where_cond += f"datetime<={end_time_str}"
             if partition_func is not None:
-                where_cond = f"int<= {partition_func(end_time)}" + ',' + where_cond
+                where_cond = f"int<= {partition_func(end_time)}" + "," + where_cond
         if other_conditions is not None:
-            where_cond = other_conditions if where_cond == "" else where_cond + ',' + other_conditions
+            where_cond = other_conditions if where_cond == "" else where_cond + "," + other_conditions
         if where_cond:
             where_cond = " where " + where_cond
         final_query = select_clause + where_cond
 
         with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
-            conn("`currpath__ set .path.pwd[]") # save current path to currpath__ as following command will change the path
+            conn(
+                "`currpath__ set .path.pwd[]"
+            )  # save current path to currpath__ as following command will change the path
             try:
                 conn(f"\\l {path}")  # load the path
                 q_table = conn(final_query)
                 return q_table.pd().set_index("datetime")
             finally:
-                conn('system "cd ", currpath__')
+                conn('system "cd ", currpath__')

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6.dev0}/trade_database_manager/core/sql/__init__.py

@@ -2,4 +2,5 @@
 # @Time    : 2024/4/15 20:28
 # @Author  : YQ Tsui
 # @File    : __init__.py
-# @Purpose :
+# @Purpose :
+from .sqlmanager import SqlManager

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6.dev0}/trade_database_manager/core/sql/sqlmanager.py

@@ -7,10 +7,11 @@
 import re
 from collections.abc import Container
 from functools import partial, reduce
-from typing import Any, Literal, Sequence, Type, Union
+from typing import Any, Literal, Sequence, Union
 
 import pandas as pd
-from sqlalchemy import DOUBLE_PRECISION, Index, Integer, MetaData, String, Table, create_engine, inspect, select, sql, text
+from sqlalchemy import Index, MetaData, Table, Column, create_engine, inspect, select, sql, text
+from sqlalchemy.types import TypeEngine
 from sqlalchemy.dialects.postgresql import insert
 
 from ...config import CONFIG
@@ -41,6 +42,13 @@ class SqlManager:
 
     def __init__(self):
         self.engine = create_engine(CONFIG["sqlconnstr"])
+        self._inspector = None
+
+    @property
+    def inspector(self):
+        if self._inspector is None:
+            self._inspector = inspect(self.engine)
+        return self._inspector
 
     def _execute(self, sql_executable: Union[str, sql.base.Executable]) -> Any:
         if isinstance(sql_executable, str):
@@ -94,8 +102,7 @@ class SqlManager:
         :rtype: int
         """
         if_exists: Literal["replace", "append"] = "append"
-        inspector = inspect(self.engine)
-        new_table = not inspector.has_table(table_name)
+        new_table = not self.inspector.has_table(table_name)
         method = partial(_insert_on_conflict_update, indexes=df.index.names) if upsert and not new_table else None
         num_rows = df.to_sql(
             table_name,
@@ -113,17 +120,44 @@ class SqlManager:
                 self.add_index(table_name, column, unique=False)
         return num_rows
 
-    @staticmethod
-    def _convert_to_sqlalchemy_type(column_type: Type, **kwargs):
-        if isinstance(column_type, type):
-            column_type = column_type.__name__
-        if column_type == "str":
-            return String(**kwargs)
-        if column_type == "int":
-            return Integer()
-        if column_type == "float":
-            return DOUBLE_PRECISION()
-        raise ValueError(f"Unsupported column type {column_type}")
+    def table_exists(self, table_name: str) -> bool:
+        """
+        Checks if a table exists.
+
+        :param table_name: The name of the table to check for.
+        :type table_name: str
+        :return: True if the table exists, False otherwise.
+        :rtype: bool
+        """
+        return self.inspector.has_table(table_name)
+
+    def create_table(
+        self,
+        table_name: str,
+        table_columns: list[tuple[str, TypeEngine]],
+        unique_index_columns: Sequence[str] = (),
+        primary_key: Union[str, set[str]] = set(),
+    ):
+        """
+        Creates a table.
+
+        :param table_name: The name of the table to create.
+        :type table_name: str
+        :param table_columns: The names of the columns to create with their data types. The keys are the column names and the values are the data types.
+        :type table_columns: dict[str, Type]
+        :param unique_index_columns: Columns to enforce unique values on. Defaults to an empty sequence.
+        :type unique_index_columns: Sequence[str]
+        :param primary_key: The primary key(s) of the table. Defaults to an empty set.
+        :type primary_key: Union[str,set[str]]
+        """
+        table_meta = MetaData()
+        if isinstance(primary_key, str):
+            primary_key = {primary_key}
+        columns = [Column(name, col_type, primary_key=name in primary_key) for name, col_type in table_columns]
+        table = Table(table_name, table_meta, *columns)
+        table.create(self.engine)
+        for column in unique_index_columns:
+            self.add_index(table_name, column, unique=True)
 
     def insert_column(self, table_name: str, column_name: str, column_type: str):
         """
@@ -314,8 +348,14 @@ class SqlManager:
         res = self._execute(stmt)
         return pd.DataFrame(res.fetchall(), columns=res.keys())
 
-    def create_ts_table(self, table_name: str, time_column: str, table_columns: dict[str, str],
-                        time_chunk_interval: str = "1 year", other_chunkers: dict[str, int] = None):
+    def create_ts_table(
+        self,
+        table_name: str,
+        time_column: str,
+        table_columns: dict[str, str],
+        time_chunk_interval: str = "1 year",
+        other_chunkers: dict[str, int] = None,
+    ):
         """
         Creates a table with a timestamp column.
 
@@ -330,19 +370,25 @@ class SqlManager:
         :param other_chunkers: Other columns to chunk by. The keys are the column names and the values are the num of hashes. Defaults to None.
         :type other_chunkers: Dict[str, int], optional
         """
-        query_create_sensordata_table  = f"""
+        query_create_sensordata_table = f"""
         CREATE TABLE {table_name} (
         {time_column} TIMESTAMP NOT NULL,
         {", ".join([f"{column_name} {column_type}{' NOT NULL' if column_name in other_chunkers else ''}" for column_name, column_type in table_columns.items()])}
         );
         """
-        query_create_sensordata_hypertable = f"SELECT create_hypertable('{table_name}', by_range('{time_column}', INTERVAL '{time_chunk_interval}'));"
+        query_create_sensordata_hypertable = (
+            f"SELECT create_hypertable('{table_name}', by_range('{time_column}', INTERVAL '{time_chunk_interval}'));"
+        )
         if other_chunkers is not None:
             for column_name, num_hash in other_chunkers.items():
-                query_create_sensordata_hypertable += f"\nSELECT add_dimension('{table_name}', by_hash('{column_name}', {num_hash}));"
+                query_create_sensordata_hypertable += (
+                    f"\nSELECT add_dimension('{table_name}', by_hash('{column_name}', {num_hash}));"
+                )
 
         uidx_name = f"uix_{table_name}_{'_'.join(other_chunkers.keys())}_{time_column}"
-        query_unique_index = f"CREATE UNIQUE INDEX {uidx_name} ON {table_name} ({', '.join(other_chunkers.keys())}, {time_column});"
+        query_unique_index = (
+            f"CREATE UNIQUE INDEX {uidx_name} ON {table_name} ({', '.join(other_chunkers.keys())}, {time_column});"
+        )
         with self.engine.connect() as conn:
             conn.execute(text(query_create_sensordata_table))
             conn.execute(text(query_create_sensordata_hypertable))

trade_database_manager-0.0.6.dev0/trade_database_manager/manager/fields_data_type.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/19 16:44
+# @Author  : YQ Tsui
+# @File    : fields_data_type.py
+# @Purpose :
+
+from sqlalchemy import DOUBLE_PRECISION, Integer, String, Date
+
+FIELD_DATA_TYPE_SQL = {
+    "ticker": String(20),
+    "name": String(20),
+    "currency": String(6),
+    "exchange": String(10),
+    "timezone": String(30),
+    "tick_size": DOUBLE_PRECISION(),
+    "lot_size": DOUBLE_PRECISION(),
+    "min_lots": DOUBLE_PRECISION(),
+    "market_tplus": Integer(),
+    "listed_date": Date(),
+    "delisted_date": Date(),
+    "country": String(6),
+    # STK
+    "sector": String(30),
+    "industry": String(36),
+    "board_type": String(200),
+    # LOF & ETF
+    "issuer": String(60),
+    "current_mgr": String(60),
+    "custodian": String(60),
+    "issuer_country": String(6),
+    "fund_type": String(20),
+    "benchmark": String(60),
+}

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6.dev0}/trade_database_manager/manager/metadata_sql.py

@@ -11,6 +11,8 @@ import pandas as pd
 
 from ..core.sql.sqlmanager import SqlManager
 from .typedefs import EXCHANGE_LITERALS, INST_TYPE_LITERALS, Opt_T_SeqT, T_DictT
+from .fields_data_type import FIELD_DATA_TYPE_SQL, String
+
 
 COMMON_METADATA_COLUMNS = [
     "name",
@@ -26,7 +28,9 @@ COMMON_METADATA_COLUMNS = [
     "delisted_date",
 ]
 TYPE_METADATA_COLUMNS = {
-    "STK": ["sector", "industry", "country", "state", "board_type", "issue_price"],
+    "STK": ["country", "state", "board_type", "issue_price"],
+    "ETF": ["issuer", "current_mgr", "custodian", "issuer_country", "fund_type", "benchmark"],
+    "LOF": ["issuer", "current_mgr", "custodian", "issuer_country", "fund_type", "benchmark"],
 }
 
 
@@ -46,6 +50,24 @@ class MetadataSql:
             cls._manager = SqlManager()
         return cls._instance
 
+    def initialize(self, for_inst_types="all"):
+        if for_inst_types == "all":
+            for_inst_types = list(TYPE_METADATA_COLUMNS.keys())
+        elif isinstance(for_inst_types, str):
+            for_inst_types = [for_inst_types]
+        columns_base = [("ticker", String(10)), ("exchange", String(10))]
+        if not self._manager.table_exists("instruments"):
+            columns = columns_base + [(col, FIELD_DATA_TYPE_SQL.get(col, String())) for col in COMMON_METADATA_COLUMNS]
+            self._manager.create_table("instruments", columns, {"primary_key": ["ticker", "exchange"]})
+        for inst_type in for_inst_types:
+            if not self._manager.table_exists(f"instruments_{inst_type.lower()}"):
+                columns = columns_base + [
+                    (col, FIELD_DATA_TYPE_SQL.get(col, String())) for col in TYPE_METADATA_COLUMNS[inst_type]
+                ]
+                self._manager.create_table(
+                    f"instruments_{inst_type.lower()}", columns, primary_key={"ticker", "exchange"}
+                )
+
     def update_instrument_metadata(self, data: Union[pd.DataFrame, list[dict], dict]):
         """
         Updates the instrument metadata in the database.
@@ -70,10 +92,12 @@ class MetadataSql:
 
         self._manager.insert("instruments", data_common, upsert=True)
         if "inst_type" in data.columns:
-            for inst_type, columns in TYPE_METADATA_COLUMNS.items():
-                data_type_df = data.loc[data.inst_type == inst_type, data.columns.intersection(columns)]
-                if not data_type_df.empty:
-                    self._manager.insert(f"instruments_{inst_type.lower()}", data_type_df, upsert=True)
+            g = data.groupby("inst_type", group_keys=False)
+            for inst_type, data_type_df in g:
+                if not data_type_df.empty and inst_type in TYPE_METADATA_COLUMNS:
+                    columns = data_type_df.columns.intersection(TYPE_METADATA_COLUMNS[inst_type])
+                    if not columns.empty:
+                        self._manager.insert(f"instruments_{inst_type.lower()}", data_type_df[columns], upsert=True)
 
     def _convert_datetime_columns(self, data: pd.DataFrame):
         for col in ["listed_date", "delisted_date"]:
@@ -132,8 +156,12 @@ class MetadataSql:
         res = {}
         for inst_type, common_df_by_type in common_df.groupby("inst_type"):
             inst_type = cast(INST_TYPE_LITERALS, inst_type)
-            if all_fields_common:
-                res[inst_type] = common_df_by_type.set_index(["ticker", "exchange"]) if len(common_df_by_type.columns) > 2 else common_df_by_type
+            if all_fields_common or inst_type not in TYPE_METADATA_COLUMNS:
+                res[inst_type] = (
+                    common_df_by_type.set_index(["ticker", "exchange"])
+                    if len(common_df_by_type.columns) > 2
+                    else common_df_by_type
+                )
                 continue
             query_fields_type = (
                 ["ticker", "exchange"] + [f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]]
@@ -187,7 +215,7 @@ class MetadataSql:
             query_fields_cross = "*"
         else:
             query_fields_common = ["ticker", "exchange"] + [f for f in query_fields if f in COMMON_METADATA_COLUMNS]
-            query_fields_type = [f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]]
+            query_fields_type = [f for f in query_fields if f in TYPE_METADATA_COLUMNS.get(inst_type, [])]
             query_fields_cross = {
                 "instruments": query_fields_common,
                 f"instruments_{inst_type.lower()}": query_fields_type,
@@ -196,19 +224,23 @@ class MetadataSql:
         filter_fields_common = {
             k: v for k, v in filter_fields.items() if k in ["ticker", "exchange"] + COMMON_METADATA_COLUMNS
         }
-        filter_fields_type = {
-            k: v for k, v in filter_fields.items() if k in ["ticker", "exchange"] + TYPE_METADATA_COLUMNS[inst_type]
-        }
-        filter_fields_cross = {
-            "instruments": filter_fields_common,
-            f"instruments_{inst_type.lower()}": filter_fields_type,
-        }
-        df = self._manager.read_data_across_tables(
-            ["instruments", f"instruments_{inst_type.lower()}"],
-            joined_columns=["ticker", "exchange"],
-            query_fields=query_fields_cross,
-            filter_fields=filter_fields_cross,
-        )
+        filter_fields_type = {k: v for k, v in filter_fields.items() if k in TYPE_METADATA_COLUMNS.get(inst_type, [])}
+
+        if (query_fields != "*" or inst_type not in TYPE_METADATA_COLUMNS) and not bool(query_fields_type) and not bool(filter_fields_type):
+            df = self._manager.read_data(
+                "instruments", query_fields=query_fields_common, filter_fields=filter_fields_common
+            )
+        else:
+            filter_fields_cross = {
+                "instruments": filter_fields_common,
+                f"instruments_{inst_type.lower()}": filter_fields_type,
+            }
+            df = self._manager.read_data_across_tables(
+                ["instruments", f"instruments_{inst_type.lower()}"],
+                joined_columns=["ticker", "exchange"],
+                query_fields=query_fields_cross,
+                filter_fields=filter_fields_cross,
+            )
 
         if isinstance(df.columns, pd.Index):
             df = df.loc[:, ~df.columns.duplicated()]

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6.dev0/trade_database_manager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.3.dev1
+Version: 0.0.6.dev0
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux

trade_database_manager-0.0.3.dev1/trade_database_manager/manager/fields_data_type.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2024/4/19 16:44
-# @Author  : YQ Tsui
-# @File    : fields_data_type.py
-# @Purpose :
-
-FIELD_DATA_TYPE_SQL = {
-    "ticker": "VARCHAR(20)",
-    "name": "VARCHAR(20)",
-    "currency": "VARCHAR(6)",
-    "exchange": "VARCHAR(10)",
-    "timezone": "VARCHAR(30)",
-    "tick_size": "REAL",
-    "lot_size": "REAL",
-    "min_lots": "REAL",
-    "market_tplus": "INTEGER",
-    "listed_date": "DATE",
-    "delisted_date": "DATE",
-    "sector": "VARCHAR(30)",
-    "industry": "VARCHAR(36)",
-    "country": "VARCHAR(36)",
-    "board_type": "VARCHAR(20)",
-}