trade-database-manager 0.0.6.dev0__tar.gz → 0.0.8__tar.gz

{trade_database_manager-0.0.6.dev0/trade_database_manager.egg-info → trade_database_manager-0.0.8}/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: trade_database_manager
-Version: 0.0.6.dev0
+Version: 0.0.8
 Summary: A wrapper of kdb and sql for convenient trade data management.
-Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
+Author-email: "Y.Q. Tsui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Development Status :: 3 - Alpha
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pandas>=2.0.0
@@ -19,6 +20,7 @@ Requires-Dist: numpy
 Requires-Dist: ruamel.yaml
 Requires-Dist: sqlalchemy
 Requires-Dist: psycopg2
+Dynamic: license-file
 
 <table border=1 cellpadding=10>
 <tr>
@@ -75,3 +77,23 @@ then the system will send an email with the license file and a base64 key (Eithe
    - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
 
    - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.
+
+### MetaData Initialization
+
+Allowed instruments types are given in "Instrument Types" section of [meta_enumerations.md](doc/meta_enumerations.md).
+
+### Data Initialization for Instrument Type(s)
+```python
+from trade_database_manager.manager import MetadataSql
+
+metadatalib = MetadataSql()
+metadatalib.initialize(for_inst_types="CB")
+```
+
+This will try to create two tables, `instruments` and `instruments_cb` in the database if not yet exists. The `instruments` table will store the common information of all instruments, and the `instruments_cb` table will store the type-specific information of the instruments of type `CB`.
+
+The table fields are listed in the [data_organization.md](doc/data_organization.md) file.
+
+
+
+

{trade_database_manager-0.0.6.dev0 → trade_database_manager-0.0.8}/README.md

@@ -53,3 +53,23 @@ then the system will send an email with the license file and a base64 key (Eithe
    - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
 
    - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.
+
+### MetaData Initialization
+
+Allowed instruments types are given in "Instrument Types" section of [meta_enumerations.md](doc/meta_enumerations.md).
+
+### Data Initialization for Instrument Type(s)
+```python
+from trade_database_manager.manager import MetadataSql
+
+metadatalib = MetadataSql()
+metadatalib.initialize(for_inst_types="CB")
+```
+
+This will try to create two tables, `instruments` and `instruments_cb` in the database if not yet exists. The `instruments` table will store the common information of all instruments, and the `instruments_cb` table will store the type-specific information of the instruments of type `CB`.
+
+The table fields are listed in the [data_organization.md](doc/data_organization.md) file.
+
+
+
+

{trade_database_manager-0.0.6.dev0 → trade_database_manager-0.0.8}/pyproject.toml

@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trade_database_manager"
-version = "0.0.6.dev0"
+version = "0.0.8"
 description = "A wrapper of kdb and sql for convenient trade data management."
 readme = "README.md"
 authors = [
-    {name = "Y.Q. Cui", email = "qianyun210603@hotmail.com"},
+    {name = "Y.Q. Tsui", email = "qianyun210603@hotmail.com"},
 ]
 dependencies = ["pandas>=2.0.0", "numpy", "ruamel.yaml", "sqlalchemy", "psycopg2"]
 classifiers = [
@@ -21,6 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 
 [tool.setuptools.packages.find]

{trade_database_manager-0.0.6.dev0 → trade_database_manager-0.0.8}/trade_database_manager/core/kdb/kdbmanager.py

@@ -4,8 +4,10 @@
 # @File    : kdbmanager.py
 # @Purpose :
 import os.path
+
 import pandas as pd
 import pykx
+
 from ...config import CONFIG
 
 

{trade_database_manager-0.0.6.dev0 → trade_database_manager-0.0.8}/trade_database_manager/core/sql/sqlmanager.py

@@ -7,15 +7,30 @@
 import re
 from collections.abc import Container
 from functools import partial, reduce
-from typing import Any, Literal, Sequence, Union
+from typing import Any, Callable, Literal, Sequence, Union
 
 import pandas as pd
-from sqlalchemy import Index, MetaData, Table, Column, create_engine, inspect, select, sql, text
-from sqlalchemy.types import TypeEngine
+from sqlalchemy import (
+    Column,
+    Executable,
+    Index,
+    MetaData,
+    PrimaryKeyConstraint,
+    Table,
+    and_,
+    create_engine,
+    func,
+    inspect,
+    select,
+    text,
+)
 from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.orm import aliased, sessionmaker
+from sqlalchemy.types import TypeEngine
 
 from ...config import CONFIG
 from ..typedefs import FILTERFIELD_TYPE, QUERYFIELD_TYPE
+from .utils import infer_sql_type
 
 
 def _insert_on_conflict_update(table, conn, keys, data_iter, indexes):
@@ -38,10 +53,13 @@ class SqlManager:
     This class is used to manage SQL operations.
 
     :ivar sqlalchemy.engine.Engine engine: An instance of the SQLAlchemy Engine class for executing SQL operations.
+    :ivar sqlalchemy.orm.Session _session: An instance of the SQLAlchemy Session class for executing SQL operations.
+    :ivar sqlalchemy.engine.reflection.Inspector _inspector: An instance of the SQLAlchemy Inspector class for inspecting the database.
     """
 
     def __init__(self):
         self.engine = create_engine(CONFIG["sqlconnstr"])
+        self._session = sessionmaker(bind=self.engine)()  # TODO: is it better to be a class attribute?
         self._inspector = None
 
     @property
@@ -50,7 +68,7 @@ class SqlManager:
             self._inspector = inspect(self.engine)
         return self._inspector
 
-    def _execute(self, sql_executable: Union[str, sql.base.Executable]) -> Any:
+    def _execute(self, sql_executable: Union[str, Executable]) -> Any:
         if isinstance(sql_executable, str):
             sql_executable = text(sql_executable)
         with self.engine.begin() as conn:
@@ -134,9 +152,9 @@ class SqlManager:
     def create_table(
         self,
         table_name: str,
-        table_columns: list[tuple[str, TypeEngine]],
+        table_columns: list[tuple[str, TypeEngine | type | tuple[type, tuple]]],
         unique_index_columns: Sequence[str] = (),
-        primary_key: Union[str, set[str]] = set(),
+        primary_key: str | set[str] = set(),
     ):
         """
         Creates a table.
@@ -148,12 +166,16 @@ class SqlManager:
         :param unique_index_columns: Columns to enforce unique values on. Defaults to an empty sequence.
         :type unique_index_columns: Sequence[str]
         :param primary_key: The primary key(s) of the table. Defaults to an empty set.
-        :type primary_key: Union[str,set[str]]
+        :type primary_key: Union[str, set[str|tuple[str]]]
+
         """
         table_meta = MetaData()
-        if isinstance(primary_key, str):
-            primary_key = {primary_key}
-        columns = [Column(name, col_type, primary_key=name in primary_key) for name, col_type in table_columns]
+        columns: list[Column|PrimaryKeyConstraint] = [Column(name, infer_sql_type(col_type)) for name, col_type in table_columns]
+        if isinstance(primary_key, str) and primary_key != "":
+            columns.append(PrimaryKeyConstraint(primary_key))
+        elif primary_key:
+            columns.append(PrimaryKeyConstraint(*primary_key, name=f"pk_{table_name}"))
+
         table = Table(table_name, table_meta, *columns)
         table.create(self.engine)
         for column in unique_index_columns:
@@ -197,7 +219,7 @@ class SqlManager:
         """
         # check if any index is referring to the column
         sql_code = f"SELECT indexname, indexdef FROM pg_indexes WHERE indexdef LIKE '%%(%%{old_column_name}%%)%%' and tablename = '{table_name}'"
-        index_refering_column = dict(self.engine.execute(sql_code).fetchall())
+        index_refering_column = dict(self._execute(sql_code).fetchall())
 
         # drop indexes referring to the column
         for index_name in index_refering_column:
@@ -219,8 +241,26 @@ class SqlManager:
         query_fields="*",
         start_time: pd.Timestamp = None,
         end_time: pd.Timestamp = None,
+        time_column: str = "timestamp",
         filter_fields=None,
     ):
+        """
+        Read data from a table within a time range.
+
+        :param table_name:
+        :type str
+        :param query_fields:
+        :type QUERYFIELD_TYPE
+        :param start_time:
+        :type pd.Timestamp
+        :param end_time:
+        :type pd.Timestamp
+        :param time_column:
+        :type str
+        :param filter_fields:
+        :type FILTERFIELD_TYPE
+        :return:
+        """
         meta = MetaData()
         table = Table(table_name, meta, autoload_with=self.engine)
 
@@ -233,9 +273,9 @@ class SqlManager:
 
         conditions = []
         if start_time:
-            conditions.append(getattr(table.c, "timestamp", table.c.end_time) >= start_time)
+            conditions.append(getattr(table.c, time_column, table.c.end_time) >= start_time)
         if end_time:
-            conditions.append(getattr(table.c, "timestamp", table.c.start_time) <= end_time)
+            conditions.append(getattr(table.c, time_column, table.c.start_time) <= end_time)
         if filter_fields:
             conditions.extend(
                 [
@@ -247,11 +287,11 @@ class SqlManager:
                     for field, filter_values in filter_fields.items()
                 ]
             )
-        stmt = stmt.where(sql.and_(*conditions))
+        stmt = stmt.where(and_(*conditions))
         res = self._execute(stmt)
         return pd.DataFrame(res.fetchall(), columns=res.keys())
 
-    def read_data(self, table_name: str, query_fields: QUERYFIELD_TYPE = "*", filter_fields=None):
+    def read_data(self, table_name: str, query_fields: QUERYFIELD_TYPE = "*", filter_fields=None, unique=False):
         """
         Reads data from a table.
 
@@ -275,6 +315,9 @@ class SqlManager:
             query_fields = table
             stmt = select(table)
 
+        if unique:
+            stmt = stmt.distinct()
+
         if filter_fields:
             conditions = [
                 (
@@ -284,7 +327,7 @@ class SqlManager:
                 )
                 for field, filter_values in filter_fields.items()
             ]
-            stmt = stmt.where(sql.and_(*conditions))
+            stmt = stmt.where(and_(*conditions))
 
         # cannot use pandas.read_sql here as it discards timezone info
         res = self._execute(stmt)
@@ -296,6 +339,7 @@ class SqlManager:
         joined_columns: Sequence[str],
         query_fields: QUERYFIELD_TYPE = "*",
         filter_fields: FILTERFIELD_TYPE = None,
+        unique = False,
     ):
         """
         Reads data from multiple tables.
@@ -315,7 +359,7 @@ class SqlManager:
         tables = {table_name: Table(table_name, meta, autoload_with=self.engine) for table_name in table_names}
 
         joined_table = reduce(
-            lambda x, y: x.join(y, sql.and_(*[x.columns[col] == y.columns[col] for col in joined_columns])),
+            lambda x, y: x.join(y, and_(*[x.columns[col] == y.columns[col] for col in joined_columns])),
             tables.values(),
         )
 
@@ -331,6 +375,9 @@ class SqlManager:
 
         stmt = select(*query_fields_rel).select_from(joined_table)
 
+        if unique:
+            stmt = stmt.distinct()
+
         if filter_fields:
             conditions = []
             for table_name, colnames in filter_fields.items():
@@ -342,55 +389,97 @@ class SqlManager:
                         else table.columns[field] == filter_values
                     )
 
-            stmt = stmt.where(sql.and_(*conditions))
+            stmt = stmt.where(and_(*conditions))
 
         # cannot use pandas.read_sql here as it discards timezone info
         res = self._execute(stmt)
         return pd.DataFrame(res.fetchall(), columns=res.keys())
 
-    def create_ts_table(
+    def _read_extremum_in_group(
         self,
         table_name: str,
-        time_column: str,
-        table_columns: dict[str, str],
-        time_chunk_interval: str = "1 year",
-        other_chunkers: dict[str, int] = None,
+        target_column: str | list[str],
+        group_column: str | list[str],
+        extremum_column: str,
+        method: str | Callable,
+        filter_fields=None,
     ):
-        """
-        Creates a table with a timestamp column.
+        if isinstance(target_column, str):
+            target_column = [target_column]
+        if isinstance(group_column, str):
+            group_column = [group_column]
+        target_column_c = [Column(col) for col in target_column]
+        group_column_c = [Column(col) for col in group_column]
+        extremum_column_c = Column(extremum_column)
 
-        :param table_name: The name of the table to create.
-        :type table_name: str
-        :param time_column: The name of the timestamp column.
-        :type time_column: str
-        :param table_columns: The names of the columns to create with their data types. The keys are the column names and the values are the data types in PostgreSQL.
-        :type table_columns: Dict[str, str]
-        :param time_chunk_interval: The interval to chunk the time column by. Defaults to "1 year".
-        :type time_chunk_interval: str
-        :param other_chunkers: Other columns to chunk by. The keys are the column names and the values are the num of hashes. Defaults to None.
-        :type other_chunkers: Dict[str, int], optional
-        """
-        query_create_sensordata_table = f"""
-        CREATE TABLE {table_name} (
-        {time_column} TIMESTAMP NOT NULL,
-        {", ".join([f"{column_name} {column_type}{' NOT NULL' if column_name in other_chunkers else ''}" for column_name, column_type in table_columns.items()])}
-        );
-        """
-        query_create_sensordata_hypertable = (
-            f"SELECT create_hypertable('{table_name}', by_range('{time_column}', INTERVAL '{time_chunk_interval}'));"
+        table_meta = MetaData()
+
+        full_table = (
+            Table(table_name, table_meta, *target_column_c, *group_column_c)
+            if extremum_column in target_column
+            else Table(table_name, table_meta, *target_column_c, *group_column_c, extremum_column_c)
         )
-        if other_chunkers is not None:
-            for column_name, num_hash in other_chunkers.items():
-                query_create_sensordata_hypertable += (
-                    f"\nSELECT add_dimension('{table_name}', by_hash('{column_name}', {num_hash}));"
+
+        if isinstance(method, str):
+            method = getattr(func, method)
+
+        if filter_fields:
+            conditions = [
+                (
+                    full_table.columns[field].in_(filter_values)
+                    if isinstance(filter_values, Container) and not isinstance(filter_values, (str, bytes))
+                    else full_table.columns[field] == filter_values
                 )
+                for field, filter_values in filter_fields.items()
+            ]
+            subquery = (
+                self._session.query(*group_column_c, method(extremum_column_c).label("ColExtremum"))
+                .where(and_(*conditions))
+                .group_by(*group_column_c)
+                .subquery()
+            )
+        else:
+            subquery = (
+                self._session.query(*group_column_c, method(extremum_column_c).label("ColExtremum"))
+                .group_by(*group_column_c)
+                .subquery()
+            )
+
+        t_full = aliased(full_table)
+        t_sub = aliased(subquery)
+
+        cols_res = [getattr(t_full.c, col) for col in group_column + target_column]
+
+        query = self._session.query(*cols_res).join(
+            t_sub,
+            and_(
+                getattr(t_full.c, extremum_column) == t_sub.c.ColExtremum,
+                *[getattr(t_full.c, col) == getattr(t_sub.c, col) for col in group_column],
+            ),
+        )
+
+        return pd.DataFrame(query.all(), columns=group_column + target_column)
 
-        uidx_name = f"uix_{table_name}_{'_'.join(other_chunkers.keys())}_{time_column}"
-        query_unique_index = (
-            f"CREATE UNIQUE INDEX {uidx_name} ON {table_name} ({', '.join(other_chunkers.keys())}, {time_column});"
+    def read_max_in_group(
+        self,
+        table_name: str,
+        target_column: str | list[str],
+        group_column: str | list[str],
+        extremum_column: str,
+        filter_fields=None,
+    ):
+        return self._read_extremum_in_group(
+            table_name, target_column, group_column, extremum_column, "max", filter_fields
+        )
+
+    def read_min_in_group(
+        self,
+        table_name: str,
+        target_column: str | list[str],
+        group_column: str | list[str],
+        extremum_column: str,
+        filter_fields=None,
+    ):
+        return self._read_extremum_in_group(
+            table_name, target_column, group_column, extremum_column, "min", filter_fields
         )
-        with self.engine.connect() as conn:
-            conn.execute(text(query_create_sensordata_table))
-            conn.execute(text(query_create_sensordata_hypertable))
-            conn.execute(text(query_unique_index))
-            conn.commit()

trade_database_manager-0.0.8/trade_database_manager/core/sql/utils.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/10/31 21:07
+# @Author  : YQ Tsui
+# @File    : utils.py
+# @Purpose : utility functions for sqlmanager
+
+from datetime import date, datetime
+
+import numpy as np
+import pandas as pd
+from sqlalchemy.types import DOUBLE_PRECISION, Date, DateTime, Integer, String, Text, TypeEngine
+
+
+def infer_sql_type(input_type):
+    if isinstance(input_type, TypeEngine):
+        return input_type
+    if isinstance(input_type, (tuple, list)):
+        input_type, *args = input_type
+    else:
+        args = ()
+    if issubclass(input_type, str):
+        if args:
+            return String(*args)
+        return Text()
+    if issubclass(input_type, (int, np.signedinteger)):
+        return Integer()
+    if issubclass(input_type, (np.floating, float)):
+        return DOUBLE_PRECISION()
+    if issubclass(input_type, (bool, np.bool_)):
+        return Integer()
+    if issubclass(input_type, (np.datetime64, pd.Timestamp, datetime)):
+        return Date() if args and args[0] else DateTime()
+    if issubclass(input_type, date):
+        return Date()
+    raise ValueError(f"Unsupported type {input_type}")

trade_database_manager-0.0.8/trade_database_manager/manager/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/21 12:41
+# @Author  : YQ Tsui
+# @File    : __init__.py
+# @Purpose :
+
+from .metadata_sql import MetadataSql
+from .metadata_sql_cb import CBMetadataSql
+from .metadata_sql_fut import FutMetadataSql
+
+__all__ = ("MetadataSql", "CBMetadataSql", "FutMetadataSql")

trade_database_manager-0.0.8/trade_database_manager/manager/fields_data_type.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/19 16:44
+# @Author  : YQ Tsui
+# @File    : fields_data_type.py
+# @Purpose :
+
+from sqlalchemy import DOUBLE_PRECISION, Date, Integer, String, Text, DateTime, CHAR
+
+FIELD_DATA_TYPE_SQL = {
+    "ticker": String(20),
+    "name": String(128),
+    "trading_code": String(20),
+    "inst_type": String(20),
+    "currency": String(6),
+    "exchange": String(10),
+    "timezone": String(30),
+    "tick_size": DOUBLE_PRECISION(),
+    "lot_size": DOUBLE_PRECISION(),
+    "min_lots": DOUBLE_PRECISION(),
+    "market_tplus": Integer(),
+    "listed_date": Date(),
+    "delisted_date": Date(),
+    "country": String(6),
+    "state": String(36),
+    # STK
+    "sector": String(30),
+    "industry": String(36),
+    "board_type": String(200),
+    # LOF & ETF
+    "issuer": String(60),
+    "current_mgr": String(60),
+    "custodian": String(60),
+    "issuer_country": String(6),
+    "fund_type": String(20),
+    "benchmark": String(60),
+    # Convertible Bond (CB)
+    "stock_ticker": String(20),
+    "stock_exchange": String(10),
+    "maturity_date": Date(),
+    "issue_price": DOUBLE_PRECISION(),
+    "total_issue_size": DOUBLE_PRECISION(),
+    "par_value": DOUBLE_PRECISION(),
+    "redeem_price": DOUBLE_PRECISION(),
+    "conversion_start_date": Date(),
+    "conversion_end_date": Date(),
+    "callback_terms": Text(),
+    "callback_type": String(20),
+    "adjust_terms": Text(),
+    "adjust_type": String(20),
+    "putback_terms": Text(),
+    "putback_type": String(20),
+    "callback_level": DOUBLE_PRECISION(),
+    # Futures (FUT)
+    "contract_unit": String(20),
+    "contract_multiplier": DOUBLE_PRECISION(),
+    "expiry_time": DateTime(),
+    "delivery_date": Date(),
+    "settlement_method": String(20),
+    "underlying_code": String(20),
+    "underlying_exchange": String(10),
+    "underlying_type": String(20),
+    "margin_method": String(20),
+    # Options (OPT), mainly similar to Futures
+    "option_type": String(20),  # 'C' for Call, 'P' for Put
+    "strike": DOUBLE_PRECISION(),
+    "exercise_style": String(20),  # 'A' for American, 'E' for European
+}
+
+DATE_TIME_COLS = {"listed_date", "delisted_date", "maturity_date", "conversion_start_date", "conversion_end_date", "expiry_date", "delivery_date"}
+
+BASE_COLUMNS = [("ticker", String(36)), ("exchange", String(10))]

{trade_database_manager-0.0.6.dev0 → trade_database_manager-0.0.8}/trade_database_manager/manager/metadata_sql.py

@@ -10,9 +10,8 @@ from typing import Union, cast
 import pandas as pd
 
 from ..core.sql.sqlmanager import SqlManager
+from .fields_data_type import BASE_COLUMNS, DATE_TIME_COLS, FIELD_DATA_TYPE_SQL
 from .typedefs import EXCHANGE_LITERALS, INST_TYPE_LITERALS, Opt_T_SeqT, T_DictT
-from .fields_data_type import FIELD_DATA_TYPE_SQL, String
-
 
 COMMON_METADATA_COLUMNS = [
     "name",
@@ -31,6 +30,40 @@ TYPE_METADATA_COLUMNS = {
     "STK": ["country", "state", "board_type", "issue_price"],
     "ETF": ["issuer", "current_mgr", "custodian", "issuer_country", "fund_type", "benchmark"],
     "LOF": ["issuer", "current_mgr", "custodian", "issuer_country", "fund_type", "benchmark"],
+    "CB": [
+        "country",
+        "state",
+        "stock_ticker",
+        "stock_exchange",
+        "maturity_date",
+        "issue_price",
+        "total_issue_size",
+        "par_value",
+        "redemption_price",
+        "conversion_start_date",
+        "conversion_end_date",
+        "callback_terms",
+        "callback_type",
+        "adjust_terms",
+        "adjust_type",
+        "putback_terms",
+        "putback_type",
+    ],
+    "FUT": ["contract_unit", "contract_multiplier", "expiry_time", "delivery_date", "settlement_method", "underlying_code", "underlying_exchange", "underlying_type", "margin_method", "sector"],
+    "OPT": [
+        "strike",
+        "option_type",
+        "exercise_style",
+        "contract_unit",
+        "contract_multiplier",
+        "expiry_time",
+        "delivery_date",
+        "settlement_method",
+        "underlying_code",
+        "underlying_exchange",
+        "underlying_type",
+        "margin_method",
+    ],
 }
 
 
@@ -55,18 +88,22 @@ class MetadataSql:
             for_inst_types = list(TYPE_METADATA_COLUMNS.keys())
         elif isinstance(for_inst_types, str):
             for_inst_types = [for_inst_types]
-        columns_base = [("ticker", String(10)), ("exchange", String(10))]
         if not self._manager.table_exists("instruments"):
-            columns = columns_base + [(col, FIELD_DATA_TYPE_SQL.get(col, String())) for col in COMMON_METADATA_COLUMNS]
+            columns = BASE_COLUMNS + [(col, FIELD_DATA_TYPE_SQL[col]) for col in COMMON_METADATA_COLUMNS]
             self._manager.create_table("instruments", columns, {"primary_key": ["ticker", "exchange"]})
         for inst_type in for_inst_types:
             if not self._manager.table_exists(f"instruments_{inst_type.lower()}"):
-                columns = columns_base + [
-                    (col, FIELD_DATA_TYPE_SQL.get(col, String())) for col in TYPE_METADATA_COLUMNS[inst_type]
-                ]
-                self._manager.create_table(
-                    f"instruments_{inst_type.lower()}", columns, primary_key={"ticker", "exchange"}
-                )
+                if inst_type not in TYPE_METADATA_COLUMNS:
+                    raise ValueError(f"Invalid instrument type: {inst_type}")
+                if inst_type in {"FUT", "OPT"}:
+                    columns = BASE_COLUMNS + [(col, FIELD_DATA_TYPE_SQL[col]) for col in (COMMON_METADATA_COLUMNS + TYPE_METADATA_COLUMNS[inst_type])]
+                    self._manager.create_table(
+                        f"instruments_{inst_type.lower()}", columns, primary_key={"ticker", "exchange"})
+                else:
+                    columns = BASE_COLUMNS + [(col, FIELD_DATA_TYPE_SQL[col]) for col in TYPE_METADATA_COLUMNS[inst_type]]
+                    self._manager.create_table(
+                        f"instruments_{inst_type.lower()}", columns, primary_key={"ticker", "exchange"}
+                    )
 
     def update_instrument_metadata(self, data: Union[pd.DataFrame, list[dict], dict]):
         """
@@ -75,11 +112,8 @@ class MetadataSql:
         :param data: The data to be updated. It can be a DataFrame, a list of dictionaries, or a single dictionary.
         :type data: Union[pd.DataFrame, list[dict], dict]
         """
-        type_specific_columns = set(data.columns) - set(COMMON_METADATA_COLUMNS)
-        if "inst_type" not in data.columns and bool(type_specific_columns):
-            raise ValueError(
-                f"Non-common columns found ({','.join(type_specific_columns)}) but inst_type column not provided."
-            )
+        if "inst_type" not in data.columns:
+            raise ValueError("`inst_type` column not provided.")
         if isinstance(data, dict):
             data = [data]
         if isinstance(data, list):
@@ -88,6 +122,19 @@ class MetadataSql:
             data.set_index(["ticker", "exchange"], inplace=True)
         else:
             assert set(data.index.names) == {"ticker", "exchange"}, "Index names must be 'ticker' and 'exchange'."
+
+        if "FUT" in data["inst_type"].unique():
+            columns = data.columns.intersection(COMMON_METADATA_COLUMNS + TYPE_METADATA_COLUMNS["FUT"])
+            data_fut = data.loc[data["inst_type"] == "FUT", columns]
+            self._manager.insert("instruments_fut", data_fut, upsert=True)
+            data.drop(index = data[data["inst_type"] == "FUT"].index, inplace=True)
+
+        if "OPT" in data["inst_type"].unique():
+            columns = data.columns.intersection(COMMON_METADATA_COLUMNS + TYPE_METADATA_COLUMNS["OPT"])
+            data_opt = data.loc[data["inst_type"] == "OPT", columns]
+            self._manager.insert("instruments_opt", data_opt, upsert=True)
+            data.drop(index = data[data["inst_type"] == "OPT"].index, inplace=True)
+
         data_common = data[data.columns.intersection(COMMON_METADATA_COLUMNS)]
 
         self._manager.insert("instruments", data_common, upsert=True)
@@ -99,11 +146,13 @@ class MetadataSql:
                     if not columns.empty:
                         self._manager.insert(f"instruments_{inst_type.lower()}", data_type_df[columns], upsert=True)
 
-    def _convert_datetime_columns(self, data: pd.DataFrame):
-        for col in ["listed_date", "delisted_date"]:
+    @staticmethod
+    def _convert_datetime_columns(data: pd.DataFrame):
+        for col in DATE_TIME_COLS:
             if col in data.columns:
                 data[col] = data[col].apply(pd.to_datetime)
 
+    # TODO: Add support for Futures and Options as they don't use the common metadata columns
     def read_metadata(
         self,
         ticker: Opt_T_SeqT[str] = None,
@@ -213,6 +262,8 @@ class MetadataSql:
 
         if query_fields == "*":
             query_fields_cross = "*"
+            query_fields_type = []
+            query_fields_common = query_fields
         else:
             query_fields_common = ["ticker", "exchange"] + [f for f in query_fields if f in COMMON_METADATA_COLUMNS]
             query_fields_type = [f for f in query_fields if f in TYPE_METADATA_COLUMNS.get(inst_type, [])]
@@ -226,7 +277,15 @@ class MetadataSql:
         }
         filter_fields_type = {k: v for k, v in filter_fields.items() if k in TYPE_METADATA_COLUMNS.get(inst_type, [])}
 
-        if (query_fields != "*" or inst_type not in TYPE_METADATA_COLUMNS) and not bool(query_fields_type) and not bool(filter_fields_type):
+        if inst_type in {"FUT", "OPT"}:
+            df = self._manager.read_data(
+                f"instruments_{inst_type.lower()}", query_fields=query_fields, filter_fields=filter_fields
+            )
+        elif (
+            (query_fields != "*" or inst_type not in TYPE_METADATA_COLUMNS)
+            and not bool(query_fields_type)
+            and not bool(filter_fields_type)
+        ):
             df = self._manager.read_data(
                 "instruments", query_fields=query_fields_common, filter_fields=filter_fields_common
             )

trade_database_manager-0.0.8/trade_database_manager/manager/metadata_sql_cb.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/11/12 13:49
+# @Author  : YQ Tsui
+# @File    : metadata_sql_cb.py
+# @Purpose :
+
+import pandas as pd
+
+from .metadata_sql import MetadataSql
+from .typedefs import EXCHANGE_LITERALS, Opt_T_SeqT, T_SeqT
+
+
+class CBMetadataSql(MetadataSql):
+
+    def read_latest_conversion_price(
+        self,
+        fields: T_SeqT[str] = ("conversion_price", "effective_date"),
+        tickers: Opt_T_SeqT[str] = None,
+        exchanges: Opt_T_SeqT[EXCHANGE_LITERALS] = None,
+        latest_by: str = "announcement_date",
+    ) -> pd.DataFrame:
+        """
+        Reads the convert price metadata from the database.
+
+        :param fields: The fields to query.
+        :type fields: T_SeqT[str]
+        :param tickers: The tickers to query, None for all.
+        :type tickers: Opt_T_SeqT[str]
+        :param exchanges: The exchanges to query, None for all.
+        :type exchanges: Opt_T_SeqT[EXCHANGE_LITERALS]
+        :param latest_by: The field to use for latest by.
+        :type latest_by: str
+        """
+        if isinstance(fields, str):
+            fields = [fields]
+        elif isinstance(fields, tuple):
+            fields = list(fields)
+        if tickers is None and exchanges is None:
+            filter_fields = None
+        else:
+            filter_fields = {}
+            if tickers is not None:
+                filter_fields["ticker"] = tickers
+            if exchanges is not None:
+                filter_fields["exchange"] = exchanges
+
+        df = self._manager.read_max_in_group(
+            "cb_convert_price_history", fields, ["ticker", "exchange"], latest_by, filter_fields=filter_fields
+        )
+        return df.set_index(["ticker", "exchange"])
+
+    def update_conversion_price(self, data: pd.DataFrame):
+        """
+        Updates the convert price metadata in the database.
+
+        :param data: The data to update.
+        :type data: pd.DataFrame
+        """
+        self._manager.insert("cb_convert_price_history", data, upsert=True)
+
+    def read_bond_coupon(
+        self, fields: T_SeqT[str], tickers: Opt_T_SeqT[str] = None, exchanges: Opt_T_SeqT[EXCHANGE_LITERALS] = None
+    ) -> pd.DataFrame:
+        """
+        Reads the convert coupon metadata from the database.
+
+        :param fields: The fields to query. available fields: ["pay_date", "coupon", "coupon_type", "period_start", "period_end", "remaining_principle", "principle_repayment"]
+        :type fields: T_SeqT[str]
+        :param tickers: The tickers to query, None for all.
+        :type tickers: Opt_T_SeqT[str]
+        :param exchanges: The exchanges to query, None for all.
+        :type exchanges: Opt_T_SeqT[EXCHANGE_LITERALS]
+        """
+        filter_fields = {}
+        if tickers is not None:
+            filter_fields["ticker"] = tickers
+        if exchanges is not None:
+            filter_fields["exchange"] = exchanges
+
+        df = self._manager.read_data("cb_coupon_schedule", query_fields=fields, filter_fields=filter_fields)
+        return df.set_index(["ticker", "exchange"])
+
+    def update_bond_coupon(self, data):
+        """
+        Updates the convert coupon metadata in the database.
+
+        :param data: The data to update.
+        :type data: pd.DataFrame
+        """
+        self._manager.insert("cb_coupon_schedule", data, upsert=True)
+
+    def update_convert_bond_cashflow(self, data):
+        """
+        Updates the convert bond cashflow metadata in the database.
+
+        :param data: The data to update.
+        :type data: pd.DataFrame
+        """
+        self._manager.insert("cb_realized_cash_flow", data, upsert=True)
+
+    def update_auxiliary(self, auxiliary_type: str, data: pd.DataFrame):
+        """
+        Updates the auxiliary data in the database.
+
+        :param auxiliary_type: ["RangerBinary"]
+        :param data: The data to update.
+        :type data: pd.DataFrame
+        """
+        self._manager.insert("cb_auxiliary_" + auxiliary_type, data, upsert=True)

trade_database_manager-0.0.8/trade_database_manager/manager/metadata_sql_fut.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/11/19 17:42
+# @Author  : YQ Tsui
+# @File    : metadata_sql_fut.py
+# @Purpose :
+
+import pandas as pd
+
+from .metadata_sql import MetadataSql
+from .typedefs import EXCHANGE_LITERALS, Opt_T_SeqT, T_SeqT
+
+class FutMetadataSql(MetadataSql):
+
+    def get_all_underlying_codes(self) -> pd.Series:
+        """
+        Get all underlying codes
+        """
+        return self._manager.read_data("instruments_fut", ["underlying_code"], unique=True)
+
+    def read_latest_daily_auxiliary_data(self, fields: T_SeqT[str] = "date",
+        tickers: Opt_T_SeqT[str] = None,
+        exchanges: Opt_T_SeqT[EXCHANGE_LITERALS] = None,
+    ) -> pd.DataFrame:
+        """
+        Read latest daily auxiliary data
+        """
+        if isinstance(fields, str):
+            fields = [fields]
+        elif isinstance(fields, tuple):
+            fields = list(fields)
+        if tickers is None and exchanges is None:
+            filter_fields = None
+        else:
+            filter_fields = {}
+            if tickers is not None:
+                filter_fields["ticker"] = tickers
+            if exchanges is not None:
+                filter_fields["exchange"] = exchanges
+
+        df = self._manager.read_max_in_group(
+            "fut_daily_auxiliary_data", fields, ["underlying_code", "exchange"], "date", filter_fields=filter_fields
+        )
+        return df.set_index(["underlying_code", "exchange"])

{trade_database_manager-0.0.6.dev0 → trade_database_manager-0.0.8}/trade_database_manager/manager/typedefs.py

@@ -6,7 +6,7 @@
 
 from typing import Dict, Literal, Optional, Sequence, TypeVar, Union
 
-INST_TYPE_LITERALS = Literal["STK", "FUT", "OPT", "IDX", "ETF", "FND", "BND", "CASH", "CRYPTO"]
+INST_TYPE_LITERALS = Literal["STK", "FUT", "OPT", "IDX", "ETF", "LOF", "FUND", "BOND", "CASH", "CRYPTO", "CB"]
 EXCHANGE_LITERALS = Literal[
     "SSE",
     "SZSE",

{trade_database_manager-0.0.6.dev0 → trade_database_manager-0.0.8/trade_database_manager.egg-info}/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: trade_database_manager
-Version: 0.0.6.dev0
+Version: 0.0.8
 Summary: A wrapper of kdb and sql for convenient trade data management.
-Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
+Author-email: "Y.Q. Tsui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Development Status :: 3 - Alpha
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pandas>=2.0.0
@@ -19,6 +20,7 @@ Requires-Dist: numpy
 Requires-Dist: ruamel.yaml
 Requires-Dist: sqlalchemy
 Requires-Dist: psycopg2
+Dynamic: license-file
 
 <table border=1 cellpadding=10>
 <tr>
@@ -75,3 +77,23 @@ then the system will send an email with the license file and a base64 key (Eithe
    - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
 
    - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.
+
+### MetaData Initialization
+
+Allowed instruments types are given in "Instrument Types" section of [meta_enumerations.md](doc/meta_enumerations.md).
+
+### Data Initialization for Instrument Type(s)
+```python
+from trade_database_manager.manager import MetadataSql
+
+metadatalib = MetadataSql()
+metadatalib.initialize(for_inst_types="CB")
+```
+
+This will try to create two tables, `instruments` and `instruments_cb` in the database if not yet exists. The `instruments` table will store the common information of all instruments, and the `instruments_cb` table will store the type-specific information of the instruments of type `CB`.
+
+The table fields are listed in the [data_organization.md](doc/data_organization.md) file.
+
+
+
+

{trade_database_manager-0.0.6.dev0 → trade_database_manager-0.0.8}/trade_database_manager.egg-info/SOURCES.txt

@@ -16,7 +16,10 @@ trade_database_manager/core/sql/__init__.py
 trade_database_manager/core/sql/sqlmanager.py
 trade_database_manager/core/sql/sqlreader.py
 trade_database_manager/core/sql/sqlwriter.py
+trade_database_manager/core/sql/utils.py
 trade_database_manager/manager/__init__.py
 trade_database_manager/manager/fields_data_type.py
 trade_database_manager/manager/metadata_sql.py
+trade_database_manager/manager/metadata_sql_cb.py
+trade_database_manager/manager/metadata_sql_fut.py
 trade_database_manager/manager/typedefs.py

trade_database_manager-0.0.6.dev0/trade_database_manager/manager/__init__.py

@@ -1,9 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2024/4/21 12:41
-# @Author  : YQ Tsui
-# @File    : __init__.py
-# @Purpose :
-
-from .metadata_sql import MetadataSql
-
-__all__ = ("MetadataSql",)

trade_database_manager-0.0.6.dev0/trade_database_manager/manager/fields_data_type.py

@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2024/4/19 16:44
-# @Author  : YQ Tsui
-# @File    : fields_data_type.py
-# @Purpose :
-
-from sqlalchemy import DOUBLE_PRECISION, Integer, String, Date
-
-FIELD_DATA_TYPE_SQL = {
-    "ticker": String(20),
-    "name": String(20),
-    "currency": String(6),
-    "exchange": String(10),
-    "timezone": String(30),
-    "tick_size": DOUBLE_PRECISION(),
-    "lot_size": DOUBLE_PRECISION(),
-    "min_lots": DOUBLE_PRECISION(),
-    "market_tplus": Integer(),
-    "listed_date": Date(),
-    "delisted_date": Date(),
-    "country": String(6),
-    # STK
-    "sector": String(30),
-    "industry": String(36),
-    "board_type": String(200),
-    # LOF & ETF
-    "issuer": String(60),
-    "current_mgr": String(60),
-    "custodian": String(60),
-    "issuer_country": String(6),
-    "fund_type": String(20),
-    "benchmark": String(60),
-}