trade-database-manager 0.0.3.dev1__tar.gz → 0.0.6__tar.gz

{trade_database_manager-0.0.3.dev1/trade_database_manager.egg-info → trade_database_manager-0.0.6}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.3.dev1
+Version: 0.0.6
 Summary: A wrapper of kdb and sql for convenient trade data management.
-Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
+Author-email: "Y.Q. Tsui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Development Status :: 3 - Alpha
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pandas>=2.0.0
@@ -75,3 +76,23 @@ then the system will send an email with the license file and a base64 key (Eithe
    - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
 
    - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.
+
+### MetaData Initialization
+
+Allowed instruments types are given in "Instrument Types" section of [meta_enumerations.md](doc/meta_enumerations.md).
+
+### Data Initialization for Instrument Type(s)
+```python
+from trade_database_manager.manager import MetadataSql
+
+metadatalib = MetadataSql()
+metadatalib.initialize(for_inst_types="CB")
+```
+
+This will try to create two tables, `instruments` and `instruments_cb` in the database if not yet exists. The `instruments` table will store the common information of all instruments, and the `instruments_cb` table will store the type-specific information of the instruments of type `CB`.
+
+The table fields are listed in the [data_organization.md](doc/data_organization.md) file.
+
+
+
+

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/README.md

@@ -53,3 +53,23 @@ then the system will send an email with the license file and a base64 key (Eithe
    - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
 
    - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.
+
+### MetaData Initialization
+
+Allowed instruments types are given in "Instrument Types" section of [meta_enumerations.md](doc/meta_enumerations.md).
+
+### Data Initialization for Instrument Type(s)
+```python
+from trade_database_manager.manager import MetadataSql
+
+metadatalib = MetadataSql()
+metadatalib.initialize(for_inst_types="CB")
+```
+
+This will try to create two tables, `instruments` and `instruments_cb` in the database if not yet exists. The `instruments` table will store the common information of all instruments, and the `instruments_cb` table will store the type-specific information of the instruments of type `CB`.
+
+The table fields are listed in the [data_organization.md](doc/data_organization.md) file.
+
+
+
+

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/pyproject.toml

@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trade_database_manager"
-version = "0.0.3.dev1"
+version = "0.0.6"
 description = "A wrapper of kdb and sql for convenient trade data management."
 readme = "README.md"
 authors = [
-    {name = "Y.Q. Cui", email = "qianyun210603@hotmail.com"},
+    {name = "Y.Q. Tsui", email = "qianyun210603@hotmail.com"},
 ]
 dependencies = ["pandas>=2.0.0", "numpy", "ruamel.yaml", "sqlalchemy", "psycopg2"]
 classifiers = [
@@ -21,6 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 
 [tool.setuptools.packages.find]
@@ -30,7 +31,7 @@ include = ["trade_database_manager*"]
 const-rgx='[a-z_][a-z0-9_]{2,30}$'
 
 [tool.pylint.main]
-disable="C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R1702,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136"
+disable="C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R1702,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R0917,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136"
 ignore-paths="\\.ipynb_checkpoints/*"
 
 [tool.pylint.format]
@@ -43,7 +44,3 @@ target-version = ['py310']
 [tool.isort]
 profile = "black"
 line_length = 120
-
-[tool.flake8]
-max-line-length = 120
-ignore = "E203,E501,W503"

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/trade_database_manager/core/kdb/__init__.py

@@ -2,4 +2,4 @@
 # @Time    : 2024/4/15 20:28
 # @Author  : YQ Tsui
 # @File    : __init__.py
-# @Purpose :
+# @Purpose :

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/trade_database_manager/core/kdb/kdbmanager.py

@@ -4,8 +4,10 @@
 # @File    : kdbmanager.py
 # @Purpose :
 import os.path
+
 import pandas as pd
 import pykx
+
 from ...config import CONFIG
 
 
@@ -17,7 +19,6 @@ class KdbManager:
     def instance(cls):
         if cls._instance is None:
             cls._instance = cls()
-            cls._instance.__init__()
         return cls._instance
 
     def __init__(self):
@@ -47,7 +48,7 @@ class KdbManager:
         :type path: str
         """
         with pykx.QConnection(self.host, self.port, username=self.username, password=self.password) as conn:
-            conn(f".path.mkdir \"{path}\"")
+            conn(f'.path.mkdir "{path}"')
 
     def write(self, table_name: str, data: pd.DataFrame, path: str = "", splayed: bool = False):
         """
@@ -69,7 +70,9 @@ class KdbManager:
         with pykx.QConnection(self.host, self.port, username=self.username, password=self.password) as conn:
             conn(f"{{`:{real_path}/ set x}}", data)
 
-    def write_partitioned(self, table_name: str, data: pd.DataFrame, path: str = "", partition_func=None, key_column=None):
+    def write_partitioned(
+        self, table_name: str, data: pd.DataFrame, path: str = "", partition_func=None, key_column=None
+    ):
         """
         Writes data to the kdb database with partitioning.
 
@@ -89,16 +92,31 @@ class KdbManager:
         assert "datetime" in data.columns, "datetime column not found"
         data.sort_values(by="datetime", inplace=True)
         if key_column is not None and key_column in data.columns:
-            with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
+            with pykx.QConnection(
+                host=self.host, port=self.port, username=self.username, password=self.password
+            ) as conn:
                 for bucket, df in data.groupby(partition_func(data["datetime"])):
-                    conn(f'{{`{table_name} set x; .partable.createOrAppend[`:{path};{bucket};`{key_column};`{table_name}]}}', df.reset_index(drop=True))
+                    conn(
+                        f"{{`{table_name} set x; .partable.createOrAppend[`:{path};{bucket};`{key_column};`{table_name}]}}",
+                        df.reset_index(drop=True),
+                    )
         else:
-            with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
+            with pykx.QConnection(
+                host=self.host, port=self.port, username=self.username, password=self.password
+            ) as conn:
                 for bucket, df in data.groupby(by=partition_func(data["datetime"])):
-                    conn(f'{{`{table_name} set x;.Q.dpt[`:{path};{bucket};`{table_name}]}}', df.reset_index(drop=True))
-
-    def read_partitioned(self, table_name: str, path: str = "", fields=None, start_time=None, end_time=None, partition_func=None,
-                         other_conditions=None):
+                    conn(f"{{`{table_name} set x;.Q.dpt[`:{path};{bucket};`{table_name}]}}", df.reset_index(drop=True))
+
+    def read_partitioned(
+        self,
+        table_name: str,
+        path: str = "",
+        fields=None,
+        start_time=None,
+        end_time=None,
+        partition_func=None,
+        other_conditions=None,
+    ):
         """
         Reads data from the kdb database with partitioning.
 
@@ -120,31 +138,39 @@ class KdbManager:
         end_time_str = end_time.strftime(time_format) if end_time is not None else None
         if isinstance(fields, (bytes, str)):
             fields = [fields]
-        select_clause = f"select from {table_name}" if fields is None else f"select {','.join(fields)} from {table_name}"
+        select_clause = (
+            f"select from {table_name}" if fields is None else f"select {','.join(fields)} from {table_name}"
+        )
         where_cond = ""
         if start_time_str is not None and end_time_str is not None:
             where_cond += f"datetime within ({start_time_str};{end_time_str})"
             if partition_func is not None:
-                where_cond = f"int in {' '.join(str(x) for x in range(partition_func(start_time), partition_func(end_time) + 1))}" + ',' + where_cond
+                where_cond = (
+                    f"int in {' '.join(str(x) for x in range(partition_func(start_time), partition_func(end_time) + 1))}"
+                    + ","
+                    + where_cond
+                )
         elif start_time_str is not None:
             where_cond += f"datetime>={start_time_str}"
             if partition_func is not None:
-                where_cond = f"int>={partition_func(start_time)}" + ',' + where_cond
+                where_cond = f"int>={partition_func(start_time)}" + "," + where_cond
         elif end_time_str is not None:
             where_cond += f"datetime<={end_time_str}"
             if partition_func is not None:
-                where_cond = f"int<= {partition_func(end_time)}" + ',' + where_cond
+                where_cond = f"int<= {partition_func(end_time)}" + "," + where_cond
         if other_conditions is not None:
-            where_cond = other_conditions if where_cond == "" else where_cond + ',' + other_conditions
+            where_cond = other_conditions if where_cond == "" else where_cond + "," + other_conditions
         if where_cond:
             where_cond = " where " + where_cond
         final_query = select_clause + where_cond
 
         with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
-            conn("`currpath__ set .path.pwd[]") # save current path to currpath__ as following command will change the path
+            conn(
+                "`currpath__ set .path.pwd[]"
+            )  # save current path to currpath__ as following command will change the path
             try:
                 conn(f"\\l {path}")  # load the path
                 q_table = conn(final_query)
                 return q_table.pd().set_index("datetime")
             finally:
-                conn('system "cd ", currpath__')
+                conn('system "cd ", currpath__')

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/trade_database_manager/core/sql/__init__.py

@@ -2,4 +2,5 @@
 # @Time    : 2024/4/15 20:28
 # @Author  : YQ Tsui
 # @File    : __init__.py
-# @Purpose :
+# @Purpose :
+from .sqlmanager import SqlManager

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/trade_database_manager/core/sql/sqlmanager.py

@@ -7,14 +7,30 @@
 import re
 from collections.abc import Container
 from functools import partial, reduce
-from typing import Any, Literal, Sequence, Type, Union
+from typing import Any, Callable, Literal, Sequence, Union
 
 import pandas as pd
-from sqlalchemy import DOUBLE_PRECISION, Index, Integer, MetaData, String, Table, create_engine, inspect, select, sql, text
+from sqlalchemy import (
+    Column,
+    Executable,
+    Index,
+    MetaData,
+    PrimaryKeyConstraint,
+    Table,
+    and_,
+    create_engine,
+    func,
+    inspect,
+    select,
+    text,
+)
 from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.orm import aliased, sessionmaker
+from sqlalchemy.types import TypeEngine
 
 from ...config import CONFIG
 from ..typedefs import FILTERFIELD_TYPE, QUERYFIELD_TYPE
+from .utils import infer_sql_type
 
 
 def _insert_on_conflict_update(table, conn, keys, data_iter, indexes):
@@ -37,12 +53,22 @@ class SqlManager:
     This class is used to manage SQL operations.
 
     :ivar sqlalchemy.engine.Engine engine: An instance of the SQLAlchemy Engine class for executing SQL operations.
+    :ivar sqlalchemy.orm.Session _session: An instance of the SQLAlchemy Session class for executing SQL operations.
+    :ivar sqlalchemy.engine.reflection.Inspector _inspector: An instance of the SQLAlchemy Inspector class for inspecting the database.
     """
 
     def __init__(self):
         self.engine = create_engine(CONFIG["sqlconnstr"])
+        self._session = sessionmaker(bind=self.engine)()  # TODO: is it better to be a class attribute?
+        self._inspector = None
 
-    def _execute(self, sql_executable: Union[str, sql.base.Executable]) -> Any:
+    @property
+    def inspector(self):
+        if self._inspector is None:
+            self._inspector = inspect(self.engine)
+        return self._inspector
+
+    def _execute(self, sql_executable: Union[str, Executable]) -> Any:
         if isinstance(sql_executable, str):
             sql_executable = text(sql_executable)
         with self.engine.begin() as conn:
@@ -94,8 +120,7 @@ class SqlManager:
         :rtype: int
         """
         if_exists: Literal["replace", "append"] = "append"
-        inspector = inspect(self.engine)
-        new_table = not inspector.has_table(table_name)
+        new_table = not self.inspector.has_table(table_name)
         method = partial(_insert_on_conflict_update, indexes=df.index.names) if upsert and not new_table else None
         num_rows = df.to_sql(
             table_name,
@@ -113,17 +138,48 @@ class SqlManager:
                 self.add_index(table_name, column, unique=False)
         return num_rows
 
-    @staticmethod
-    def _convert_to_sqlalchemy_type(column_type: Type, **kwargs):
-        if isinstance(column_type, type):
-            column_type = column_type.__name__
-        if column_type == "str":
-            return String(**kwargs)
-        if column_type == "int":
-            return Integer()
-        if column_type == "float":
-            return DOUBLE_PRECISION()
-        raise ValueError(f"Unsupported column type {column_type}")
+    def table_exists(self, table_name: str) -> bool:
+        """
+        Checks if a table exists.
+
+        :param table_name: The name of the table to check for.
+        :type table_name: str
+        :return: True if the table exists, False otherwise.
+        :rtype: bool
+        """
+        return self.inspector.has_table(table_name)
+
+    def create_table(
+        self,
+        table_name: str,
+        table_columns: list[tuple[str, TypeEngine | type | tuple[type, tuple]]],
+        unique_index_columns: Sequence[str] = (),
+        primary_key: str | set[str] = set(),
+    ):
+        """
+        Creates a table.
+
+        :param table_name: The name of the table to create.
+        :type table_name: str
+        :param table_columns: The names of the columns to create with their data types. The keys are the column names and the values are the data types.
+        :type table_columns: dict[str, Type]
+        :param unique_index_columns: Columns to enforce unique values on. Defaults to an empty sequence.
+        :type unique_index_columns: Sequence[str]
+        :param primary_key: The primary key(s) of the table. Defaults to an empty set.
+        :type primary_key: Union[str, set[str|tuple[str]]]
+
+        """
+        table_meta = MetaData()
+        columns = [Column(name, infer_sql_type(col_type)) for name, col_type in table_columns]
+        if isinstance(primary_key, str) and primary_key != "":
+            columns.append(PrimaryKeyConstraint(primary_key))
+        elif primary_key:
+            columns.append(PrimaryKeyConstraint(*primary_key, name=f"pk_{table_name}"))
+
+        table = Table(table_name, table_meta, *columns)
+        table.create(self.engine)
+        for column in unique_index_columns:
+            self.add_index(table_name, column, unique=True)
 
     def insert_column(self, table_name: str, column_name: str, column_type: str):
         """
@@ -163,7 +219,7 @@ class SqlManager:
         """
         # check if any index is referring to the column
         sql_code = f"SELECT indexname, indexdef FROM pg_indexes WHERE indexdef LIKE '%%(%%{old_column_name}%%)%%' and tablename = '{table_name}'"
-        index_refering_column = dict(self.engine.execute(sql_code).fetchall())
+        index_refering_column = dict(self._execute(sql_code).fetchall())
 
         # drop indexes referring to the column
         for index_name in index_refering_column:
@@ -213,7 +269,7 @@ class SqlManager:
                     for field, filter_values in filter_fields.items()
                 ]
             )
-        stmt = stmt.where(sql.and_(*conditions))
+        stmt = stmt.where(and_(*conditions))
         res = self._execute(stmt)
         return pd.DataFrame(res.fetchall(), columns=res.keys())
 
@@ -250,7 +306,7 @@ class SqlManager:
                 )
                 for field, filter_values in filter_fields.items()
             ]
-            stmt = stmt.where(sql.and_(*conditions))
+            stmt = stmt.where(and_(*conditions))
 
         # cannot use pandas.read_sql here as it discards timezone info
         res = self._execute(stmt)
@@ -281,7 +337,7 @@ class SqlManager:
         tables = {table_name: Table(table_name, meta, autoload_with=self.engine) for table_name in table_names}
 
         joined_table = reduce(
-            lambda x, y: x.join(y, sql.and_(*[x.columns[col] == y.columns[col] for col in joined_columns])),
+            lambda x, y: x.join(y, and_(*[x.columns[col] == y.columns[col] for col in joined_columns])),
             tables.values(),
         )
 
@@ -308,43 +364,97 @@ class SqlManager:
                         else table.columns[field] == filter_values
                     )
 
-            stmt = stmt.where(sql.and_(*conditions))
+            stmt = stmt.where(and_(*conditions))
 
         # cannot use pandas.read_sql here as it discards timezone info
         res = self._execute(stmt)
         return pd.DataFrame(res.fetchall(), columns=res.keys())
 
-    def create_ts_table(self, table_name: str, time_column: str, table_columns: dict[str, str],
-                        time_chunk_interval: str = "1 year", other_chunkers: dict[str, int] = None):
-        """
-        Creates a table with a timestamp column.
+    def _read_extremum_in_group(
+        self,
+        table_name: str,
+        target_column: str | list[str],
+        group_column: str | list[str],
+        extremum_column: str,
+        method: str | Callable,
+        filter_fields=None,
+    ):
+        if isinstance(target_column, str):
+            target_column = [target_column]
+        if isinstance(group_column, str):
+            group_column = [group_column]
+        target_column_c = [Column(col) for col in target_column]
+        group_column_c = [Column(col) for col in group_column]
+        extremum_column_c = Column(extremum_column)
 
-        :param table_name: The name of the table to create.
-        :type table_name: str
-        :param time_column: The name of the timestamp column.
-        :type time_column: str
-        :param table_columns: The names of the columns to create with their data types. The keys are the column names and the values are the data types in PostgreSQL.
-        :type table_columns: Dict[str, str]
-        :param time_chunk_interval: The interval to chunk the time column by. Defaults to "1 year".
-        :type time_chunk_interval: str
-        :param other_chunkers: Other columns to chunk by. The keys are the column names and the values are the num of hashes. Defaults to None.
-        :type other_chunkers: Dict[str, int], optional
-        """
-        query_create_sensordata_table  = f"""
-        CREATE TABLE {table_name} (
-        {time_column} TIMESTAMP NOT NULL,
-        {", ".join([f"{column_name} {column_type}{' NOT NULL' if column_name in other_chunkers else ''}" for column_name, column_type in table_columns.items()])}
-        );
-        """
-        query_create_sensordata_hypertable = f"SELECT create_hypertable('{table_name}', by_range('{time_column}', INTERVAL '{time_chunk_interval}'));"
-        if other_chunkers is not None:
-            for column_name, num_hash in other_chunkers.items():
-                query_create_sensordata_hypertable += f"\nSELECT add_dimension('{table_name}', by_hash('{column_name}', {num_hash}));"
-
-        uidx_name = f"uix_{table_name}_{'_'.join(other_chunkers.keys())}_{time_column}"
-        query_unique_index = f"CREATE UNIQUE INDEX {uidx_name} ON {table_name} ({', '.join(other_chunkers.keys())}, {time_column});"
-        with self.engine.connect() as conn:
-            conn.execute(text(query_create_sensordata_table))
-            conn.execute(text(query_create_sensordata_hypertable))
-            conn.execute(text(query_unique_index))
-            conn.commit()
+        table_meta = MetaData()
+
+        full_table = (
+            Table(table_name, table_meta, *target_column_c, *group_column_c)
+            if extremum_column in target_column
+            else Table(table_name, table_meta, *target_column_c, *group_column_c, extremum_column_c)
+        )
+
+        if isinstance(method, str):
+            method = getattr(func, method)
+
+        if filter_fields:
+            conditions = [
+                (
+                    full_table.columns[field].in_(filter_values)
+                    if isinstance(filter_values, Container) and not isinstance(filter_values, (str, bytes))
+                    else full_table.columns[field] == filter_values
+                )
+                for field, filter_values in filter_fields.items()
+            ]
+            subquery = (
+                self._session.query(*group_column_c, method(extremum_column_c).label("ColExtremum"))
+                .where(and_(*conditions))
+                .group_by(*group_column_c)
+                .subquery()
+            )
+        else:
+            subquery = (
+                self._session.query(*group_column_c, method(extremum_column_c).label("ColExtremum"))
+                .group_by(*group_column_c)
+                .subquery()
+            )
+
+        t_full = aliased(full_table)
+        t_sub = aliased(subquery)
+
+        cols_res = [getattr(t_full.c, col) for col in group_column + target_column]
+
+        query = self._session.query(*cols_res).join(
+            t_sub,
+            and_(
+                getattr(t_full.c, extremum_column) == t_sub.c.ColExtremum,
+                *[getattr(t_full.c, col) == getattr(t_sub.c, col) for col in group_column],
+            ),
+        )
+
+        return pd.DataFrame(query.all(), columns=group_column + target_column)
+
+    def read_max_in_group(
+        self,
+        table_name: str,
+        target_column: str | list[str],
+        group_column: str | list[str],
+        extremum_column: str,
+        filter_fields=None,
+    ):
+        return self._read_extremum_in_group(
+            table_name, target_column, group_column, extremum_column, "max", filter_fields
+        )
+
+    def read_min_in_group(
+        self,
+        table_name: str,
+        target_column: str | list[str],
+        group_column: str | list[str],
+        extremum_column: str,
+        filter_fields=None,
+    ):
+        return self._read_extremum_in_group(
+            table_name, target_column, group_column, extremum_column, "min", filter_fields
+        )

trade_database_manager-0.0.6/trade_database_manager/core/sql/utils.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/10/31 21:07
+# @Author  : YQ Tsui
+# @File    : utils.py
+# @Purpose : utility functions for sqlmanager
+
+from datetime import date, datetime
+
+import numpy as np
+import pandas as pd
+from sqlalchemy.types import DOUBLE_PRECISION, Date, DateTime, Integer, String, Text, TypeEngine
+
+
+def infer_sql_type(input_type):
+    if isinstance(input_type, TypeEngine):
+        return input_type
+    if isinstance(input_type, (tuple, list)):
+        input_type, *args = input_type
+    else:
+        args = ()
+    if issubclass(input_type, str):
+        if args:
+            return String(*args)
+        return Text()
+    if issubclass(input_type, (int, np.signedinteger)):
+        return Integer()
+    if issubclass(input_type, (np.floating, float)):
+        return DOUBLE_PRECISION()
+    if issubclass(input_type, (bool, np.bool_)):
+        return Integer()
+    if issubclass(input_type, (np.datetime64, pd.Timestamp, datetime)):
+        return Date() if args and args[0] else DateTime()
+    if issubclass(input_type, date):
+        return Date()
+    raise ValueError(f"Unsupported type {input_type}")

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/trade_database_manager/manager/__init__.py

@@ -5,5 +5,6 @@
 # @Purpose :
 
 from .metadata_sql import MetadataSql
+from .metadata_sql_cb import CBMetadataSql
 
-__all__ = ("MetadataSql",)
+__all__ = ("MetadataSql", "CBMetadataSql")

trade_database_manager-0.0.6/trade_database_manager/manager/fields_data_type.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/19 16:44
+# @Author  : YQ Tsui
+# @File    : fields_data_type.py
+# @Purpose :
+
+from sqlalchemy import DOUBLE_PRECISION, Date, Integer, String, Text
+
+FIELD_DATA_TYPE_SQL = {
+    "ticker": String(20),
+    "name": String(20),
+    "currency": String(6),
+    "exchange": String(10),
+    "timezone": String(30),
+    "tick_size": DOUBLE_PRECISION(),
+    "lot_size": DOUBLE_PRECISION(),
+    "min_lots": DOUBLE_PRECISION(),
+    "market_tplus": Integer(),
+    "listed_date": Date(),
+    "delisted_date": Date(),
+    "country": String(6),
+    "state": String(36),
+    # STK
+    "sector": String(30),
+    "industry": String(36),
+    "board_type": String(200),
+    # LOF & ETF
+    "issuer": String(60),
+    "current_mgr": String(60),
+    "custodian": String(60),
+    "issuer_country": String(6),
+    "fund_type": String(20),
+    "benchmark": String(60),
+    # Convertible Bond (CB)
+    "stock_ticker": String(20),
+    "stock_exchange": String(10),
+    "maturity_date": Date(),
+    "issue_price": DOUBLE_PRECISION(),
+    "total_issue_size": DOUBLE_PRECISION(),
+    "par_value": DOUBLE_PRECISION(),
+    "redeem_price": DOUBLE_PRECISION(),
+    "conversion_start_date": Date(),
+    "conversion_end_date": Date(),
+    "callback_terms": Text(),
+    "callback_type": String(20),
+    "adjust_terms": Text(),
+    "adjust_type": String(20),
+    "putback_terms": Text(),
+    "putback_type": String(20),
+    "callback_level": DOUBLE_PRECISION(),
+}
+
+DATE_TIME_COLS = {"listed_date", "delisted_date", "maturity_date", "conversion_start_date", "conversion_end_date"}
+
+BASE_COLUMNS = [("ticker", String(10)), ("exchange", String(10))]

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/trade_database_manager/manager/metadata_sql.py

@@ -10,6 +10,7 @@ from typing import Union, cast
 import pandas as pd
 
 from ..core.sql.sqlmanager import SqlManager
+from .fields_data_type import BASE_COLUMNS, DATE_TIME_COLS, FIELD_DATA_TYPE_SQL
 from .typedefs import EXCHANGE_LITERALS, INST_TYPE_LITERALS, Opt_T_SeqT, T_DictT
 
 COMMON_METADATA_COLUMNS = [
@@ -26,7 +27,28 @@ COMMON_METADATA_COLUMNS = [
     "delisted_date",
 ]
 TYPE_METADATA_COLUMNS = {
-    "STK": ["sector", "industry", "country", "state", "board_type", "issue_price"],
+    "STK": ["country", "state", "board_type", "issue_price"],
+    "ETF": ["issuer", "current_mgr", "custodian", "issuer_country", "fund_type", "benchmark"],
+    "LOF": ["issuer", "current_mgr", "custodian", "issuer_country", "fund_type", "benchmark"],
+    "CB": [
+        "country",
+        "state",
+        "stock_ticker",
+        "stock_exchange",
+        "maturity_date",
+        "issue_price",
+        "total_issue_size",
+        "par_value",
+        "redemption_price",
+        "conversion_start_date",
+        "conversion_end_date",
+        "callback_terms",
+        "callback_type",
+        "adjust_terms",
+        "adjust_type",
+        "putback_terms",
+        "putback_type",
+    ],
 }
 
 
@@ -46,6 +68,21 @@ class MetadataSql:
             cls._manager = SqlManager()
         return cls._instance
 
+    def initialize(self, for_inst_types="all"):
+        if for_inst_types == "all":
+            for_inst_types = list(TYPE_METADATA_COLUMNS.keys())
+        elif isinstance(for_inst_types, str):
+            for_inst_types = [for_inst_types]
+        if not self._manager.table_exists("instruments"):
+            columns = BASE_COLUMNS + [(col, FIELD_DATA_TYPE_SQL[col]) for col in COMMON_METADATA_COLUMNS]
+            self._manager.create_table("instruments", columns, {"primary_key": ["ticker", "exchange"]})
+        for inst_type in for_inst_types:
+            if not self._manager.table_exists(f"instruments_{inst_type.lower()}"):
+                columns = BASE_COLUMNS + [(col, FIELD_DATA_TYPE_SQL[col]) for col in TYPE_METADATA_COLUMNS[inst_type]]
+                self._manager.create_table(
+                    f"instruments_{inst_type.lower()}", columns, primary_key={"ticker", "exchange"}
+                )
+
     def update_instrument_metadata(self, data: Union[pd.DataFrame, list[dict], dict]):
         """
         Updates the instrument metadata in the database.
@@ -70,13 +107,16 @@ class MetadataSql:
 
         self._manager.insert("instruments", data_common, upsert=True)
         if "inst_type" in data.columns:
-            for inst_type, columns in TYPE_METADATA_COLUMNS.items():
-                data_type_df = data.loc[data.inst_type == inst_type, data.columns.intersection(columns)]
-                if not data_type_df.empty:
-                    self._manager.insert(f"instruments_{inst_type.lower()}", data_type_df, upsert=True)
+            g = data.groupby("inst_type", group_keys=False)
+            for inst_type, data_type_df in g:
+                if not data_type_df.empty and inst_type in TYPE_METADATA_COLUMNS:
+                    columns = data_type_df.columns.intersection(TYPE_METADATA_COLUMNS[inst_type])
+                    if not columns.empty:
+                        self._manager.insert(f"instruments_{inst_type.lower()}", data_type_df[columns], upsert=True)
 
-    def _convert_datetime_columns(self, data: pd.DataFrame):
-        for col in ["listed_date", "delisted_date"]:
+    @staticmethod
+    def _convert_datetime_columns(data: pd.DataFrame):
+        for col in DATE_TIME_COLS:
             if col in data.columns:
                 data[col] = data[col].apply(pd.to_datetime)
 
@@ -132,8 +172,12 @@ class MetadataSql:
         res = {}
         for inst_type, common_df_by_type in common_df.groupby("inst_type"):
             inst_type = cast(INST_TYPE_LITERALS, inst_type)
-            if all_fields_common:
-                res[inst_type] = common_df_by_type.set_index(["ticker", "exchange"]) if len(common_df_by_type.columns) > 2 else common_df_by_type
+            if all_fields_common or inst_type not in TYPE_METADATA_COLUMNS:
+                res[inst_type] = (
+                    common_df_by_type.set_index(["ticker", "exchange"])
+                    if len(common_df_by_type.columns) > 2
+                    else common_df_by_type
+                )
                 continue
             query_fields_type = (
                 ["ticker", "exchange"] + [f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]]
@@ -185,9 +229,11 @@ class MetadataSql:
 
         if query_fields == "*":
             query_fields_cross = "*"
+            query_fields_type = []
+            query_fields_common = query_fields
         else:
             query_fields_common = ["ticker", "exchange"] + [f for f in query_fields if f in COMMON_METADATA_COLUMNS]
-            query_fields_type = [f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]]
+            query_fields_type = [f for f in query_fields if f in TYPE_METADATA_COLUMNS.get(inst_type, [])]
             query_fields_cross = {
                 "instruments": query_fields_common,
                 f"instruments_{inst_type.lower()}": query_fields_type,
@@ -196,19 +242,27 @@ class MetadataSql:
         filter_fields_common = {
             k: v for k, v in filter_fields.items() if k in ["ticker", "exchange"] + COMMON_METADATA_COLUMNS
         }
-        filter_fields_type = {
-            k: v for k, v in filter_fields.items() if k in ["ticker", "exchange"] + TYPE_METADATA_COLUMNS[inst_type]
-        }
-        filter_fields_cross = {
-            "instruments": filter_fields_common,
-            f"instruments_{inst_type.lower()}": filter_fields_type,
-        }
-        df = self._manager.read_data_across_tables(
-            ["instruments", f"instruments_{inst_type.lower()}"],
-            joined_columns=["ticker", "exchange"],
-            query_fields=query_fields_cross,
-            filter_fields=filter_fields_cross,
-        )
+        filter_fields_type = {k: v for k, v in filter_fields.items() if k in TYPE_METADATA_COLUMNS.get(inst_type, [])}
+
+        if (
+            (query_fields != "*" or inst_type not in TYPE_METADATA_COLUMNS)
+            and not bool(query_fields_type)
+            and not bool(filter_fields_type)
+        ):
+            df = self._manager.read_data(
+                "instruments", query_fields=query_fields_common, filter_fields=filter_fields_common
+            )
+        else:
+            filter_fields_cross = {
+                "instruments": filter_fields_common,
+                f"instruments_{inst_type.lower()}": filter_fields_type,
+            }
+            df = self._manager.read_data_across_tables(
+                ["instruments", f"instruments_{inst_type.lower()}"],
+                joined_columns=["ticker", "exchange"],
+                query_fields=query_fields_cross,
+                filter_fields=filter_fields_cross,
+            )
 
         if isinstance(df.columns, pd.Index):
             df = df.loc[:, ~df.columns.duplicated()]

trade_database_manager-0.0.6/trade_database_manager/manager/metadata_sql_cb.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/11/12 13:49
+# @Author  : YQ Tsui
+# @File    : metadata_sql_cb.py
+# @Purpose :
+
+import pandas as pd
+
+from .metadata_sql import MetadataSql
+from .typedefs import EXCHANGE_LITERALS, Opt_T_SeqT, T_SeqT
+
+
+class CBMetadataSql(MetadataSql):
+
+    def read_latest_conversion_price(
+        self,
+        fields: T_SeqT[str] = ("conversion_price", "effective_date"),
+        tickers: Opt_T_SeqT[str] = None,
+        exchanges: Opt_T_SeqT[EXCHANGE_LITERALS] = None,
+        latest_by: str = "announcement_date",
+    ) -> pd.DataFrame:
+        """
+        Reads the convert price metadata from the database.
+
+        :param fields: The fields to query.
+        :type fields: T_SeqT[str]
+        :param tickers: The tickers to query, None for all.
+        :type tickers: Opt_T_SeqT[str]
+        :param exchanges: The exchanges to query, None for all.
+        :type exchanges: Opt_T_SeqT[EXCHANGE_LITERALS]
+        :param latest_by: The field to use for latest by.
+        :type latest_by: str
+        """
+        if isinstance(fields, str):
+            fields = [fields]
+        elif isinstance(fields, tuple):
+            fields = list(fields)
+        if tickers is None and exchanges is None:
+            filter_fields = None
+        else:
+            filter_fields = {}
+            if tickers is not None:
+                filter_fields["ticker"] = tickers
+            if exchanges is not None:
+                filter_fields["exchange"] = exchanges
+
+        df = self._manager.read_max_in_group(
+            "cb_convert_price_history", fields, ["ticker", "exchange"], latest_by, filter_fields=filter_fields
+        )
+        return df.set_index(["ticker", "exchange"])
+
+    def update_conversion_price(self, data: pd.DataFrame):
+        """
+        Updates the convert price metadata in the database.
+
+        :param data: The data to update.
+        :type data: pd.DataFrame
+        """
+        self._manager.insert("cb_convert_price_history", data, upsert=True)
+
+    def read_bond_coupon(
+        self, fields: T_SeqT[str], tickers: Opt_T_SeqT[str] = None, exchanges: Opt_T_SeqT[EXCHANGE_LITERALS] = None
+    ) -> pd.DataFrame:
+        """
+        Reads the convert coupon metadata from the database.
+
+        :param fields: The fields to query. available fields: ["pay_date", "coupon", "coupon_type", "period_start", "period_end", "remaining_principle", "principle_repayment"]
+        :type fields: T_SeqT[str]
+        :param tickers: The tickers to query, None for all.
+        :type tickers: Opt_T_SeqT[str]
+        :param exchanges: The exchanges to query, None for all.
+        :type exchanges: Opt_T_SeqT[EXCHANGE_LITERALS]
+        """
+        filter_fields = {}
+        if tickers is not None:
+            filter_fields["ticker"] = tickers
+        if exchanges is not None:
+            filter_fields["exchange"] = exchanges
+
+        df = self._manager.read_data("cb_coupon_schedule", query_fields=fields, filter_fields=filter_fields)
+        return df.set_index(["ticker", "exchange"])
+
+    def update_bond_coupon(self, data):
+        """
+        Updates the convert coupon metadata in the database.
+
+        :param data: The data to update.
+        :type data: pd.DataFrame
+        """
+        self._manager.insert("cb_coupon_schedule", data, upsert=True)
+
+    def update_convert_bond_cashflow(self, data):
+        """
+        Updates the convert bond cashflow metadata in the database.
+
+        :param data: The data to update.
+        :type data: pd.DataFrame
+        """
+        self._manager.insert("cb_realized_cash_flow", data, upsert=True)
+
+    def update_auxiliary(self, auxiliary_type: str, data: pd.DataFrame):
+        """
+        Updates the auxiliary data in the database.
+
+        :param data: The data to update.
+        :type data: pd.DataFrame
+        """
+        self._manager.insert("cb_auxiliary_" + auxiliary_type, data, upsert=True)

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/trade_database_manager/manager/typedefs.py

@@ -6,7 +6,7 @@
 
 from typing import Dict, Literal, Optional, Sequence, TypeVar, Union
 
-INST_TYPE_LITERALS = Literal["STK", "FUT", "OPT", "IDX", "ETF", "FND", "BND", "CASH", "CRYPTO"]
+INST_TYPE_LITERALS = Literal["STK", "FUT", "OPT", "IDX", "ETF", "LOF", "FUND", "BOND", "CASH", "CRYPTO", "CB"]
 EXCHANGE_LITERALS = Literal[
     "SSE",
     "SZSE",

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6/trade_database_manager.egg-info}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.3.dev1
+Version: 0.0.6
 Summary: A wrapper of kdb and sql for convenient trade data management.
-Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
+Author-email: "Y.Q. Tsui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Development Status :: 3 - Alpha
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pandas>=2.0.0
@@ -75,3 +76,23 @@ then the system will send an email with the license file and a base64 key (Eithe
    - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
 
    - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.
+
+### MetaData Initialization
+
+Allowed instruments types are given in "Instrument Types" section of [meta_enumerations.md](doc/meta_enumerations.md).
+
+### Data Initialization for Instrument Type(s)
+```python
+from trade_database_manager.manager import MetadataSql
+
+metadatalib = MetadataSql()
+metadatalib.initialize(for_inst_types="CB")
+```
+
+This will try to create two tables, `instruments` and `instruments_cb` in the database if not yet exists. The `instruments` table will store the common information of all instruments, and the `instruments_cb` table will store the type-specific information of the instruments of type `CB`.
+
+The table fields are listed in the [data_organization.md](doc/data_organization.md) file.
+
+
+
+

{trade_database_manager-0.0.3.dev1 → trade_database_manager-0.0.6}/trade_database_manager.egg-info/SOURCES.txt

@@ -16,7 +16,9 @@ trade_database_manager/core/sql/__init__.py
 trade_database_manager/core/sql/sqlmanager.py
 trade_database_manager/core/sql/sqlreader.py
 trade_database_manager/core/sql/sqlwriter.py
+trade_database_manager/core/sql/utils.py
 trade_database_manager/manager/__init__.py
 trade_database_manager/manager/fields_data_type.py
 trade_database_manager/manager/metadata_sql.py
+trade_database_manager/manager/metadata_sql_cb.py
 trade_database_manager/manager/typedefs.py

trade_database_manager-0.0.3.dev1/trade_database_manager/manager/fields_data_type.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2024/4/19 16:44
-# @Author  : YQ Tsui
-# @File    : fields_data_type.py
-# @Purpose :
-
-FIELD_DATA_TYPE_SQL = {
-    "ticker": "VARCHAR(20)",
-    "name": "VARCHAR(20)",
-    "currency": "VARCHAR(6)",
-    "exchange": "VARCHAR(10)",
-    "timezone": "VARCHAR(30)",
-    "tick_size": "REAL",
-    "lot_size": "REAL",
-    "min_lots": "REAL",
-    "market_tplus": "INTEGER",
-    "listed_date": "DATE",
-    "delisted_date": "DATE",
-    "sector": "VARCHAR(30)",
-    "industry": "VARCHAR(36)",
-    "country": "VARCHAR(36)",
-    "board_type": "VARCHAR(20)",
-}