trade-database-manager 0.0.1.dev1__tar.gz → 0.0.1.dev4__tar.gz

{trade_database_manager-0.0.1.dev1/trade_database_manager.egg-info → trade_database_manager-0.0.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.1.dev1
+Version: 0.0.1.dev4
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux
@@ -52,5 +52,26 @@ host    all            tradedbadmin     ::/0                    scram-sha-256
 
 ### KDB+ Initialization
 
+#### get license
+Request a 64-bit license from [Kx Systems](https://kx.com/kdb-insights-personal-edition-license-download). Fill the form
+then the system will send an email with the license file and a base64 key (Either can be used to license the product).
 
-```q
+#### Server side (Linux system)
+1. Put the executable `q` and license in desired folder, say `/opt/l64` directory.
+2. Create a folder to store the data, say `/opt/data` directory.
+3. Set the environment variable `QLIC` to the directory where the license file is stored. And `QHOME` to the data directory.
+4. Run the `q` executable with the following command:
+    ```bash
+    $ q -p 5000 -s 1 -w 100 -t 1000 -T 1000 -U /opt/l64/trade.q
+    ```
+   - `-p 5000`: This sets the port number for the kdb+ process. In this case, the port number is 5000.
+
+   - `-s 1`: This sets the number of secondary threads. In this case, it's set to 1.
+
+   - `-w 100`: This sets the workspace heap size. In this case, it's set to 100 MB.
+
+   - `-t 1000`: This sets the timer interval in milliseconds. In this case, it's set to 1000 milliseconds, or 1 second.
+
+   - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
+
+   - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.

trade_database_manager-0.0.1.dev4/README.md

@@ -0,0 +1,55 @@
+<table border=1 cellpadding=10>
+<tr>
+<td style="color: red;">
+
+#### \*\*\* IMPORTANT NOTICE \*\*\*
+
+<p style="color: red">This package is <b>not</b> in a usable stage. It is only uploaded for convenience of developing and testing.</p>
+
+</td></tr></table>
+
+
+
+### Postgreslq Initialization
+
+```sql
+CREATE DATABASE trade_data;
+CREATE USER tradedbadmin WITH PASSWORD 'trade_password';
+GRANT ALL PRIVILEGES ON DATABASE trade_data TO tradedbadmin;
+\connect trade_data;
+GRANT ALL ON SCHEMA public TO tradedbadmin;
+```
+
+Add the following lines in the `pg_hba.conf` file to allow password authentication for the `tradedbadmin` role.
+
+```
+# trade database
+host    all            tradedbadmin     0.0.0.0/0               scram-sha-256
+host    all            tradedbadmin     ::/0                    scram-sha-256
+```
+
+### KDB+ Initialization
+
+#### get license
+Request a 64-bit license from [Kx Systems](https://kx.com/kdb-insights-personal-edition-license-download). Fill the form
+then the system will send an email with the license file and a base64 key (Either can be used to license the product).
+
+#### Server side (Linux system)
+1. Put the executable `q` and license in desired folder, say `/opt/l64` directory.
+2. Create a folder to store the data, say `/opt/data` directory.
+3. Set the environment variable `QLIC` to the directory where the license file is stored. And `QHOME` to the data directory.
+4. Run the `q` executable with the following command:
+    ```bash
+    $ q -p 5000 -s 1 -w 100 -t 1000 -T 1000 -U /opt/l64/trade.q
+    ```
+   - `-p 5000`: This sets the port number for the kdb+ process. In this case, the port number is 5000.
+
+   - `-s 1`: This sets the number of secondary threads. In this case, it's set to 1.
+
+   - `-w 100`: This sets the workspace heap size. In this case, it's set to 100 MB.
+
+   - `-t 1000`: This sets the timer interval in milliseconds. In this case, it's set to 1000 milliseconds, or 1 second.
+
+   - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
+
+   - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.

{trade_database_manager-0.0.1.dev1 → trade_database_manager-0.0.1.dev4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trade_database_manager"
-version = "0.0.1.dev1"
+version = "0.0.1.dev4"
 description = "A wrapper of kdb and sql for convenient trade data management."
 readme = "README.md"
 authors = [
@@ -42,3 +42,8 @@ target-version = ['py310']
 
 [tool.isort]
 profile = "black"
+line_length = 120
+
+[tool.flake8]
+max-line-length = 120
+ignore = "E203,E501,W503"

{trade_database_manager-0.0.1.dev1 → trade_database_manager-0.0.1.dev4}/setup.py

@@ -1,4 +1,3 @@
 from setuptools import setup
 
-
 setup()

trade_database_manager-0.0.1.dev4/trade_database_manager/core/sql/sqlmanager.py

@@ -0,0 +1,312 @@
+import re
+from collections.abc import Container
+from functools import partial, reduce
+from typing import Any, Literal, Sequence, Type, Union
+
+import pandas as pd
+from sqlalchemy import REAL, Index, Integer, MetaData, String, Table, create_engine, inspect, select, sql, text
+from sqlalchemy.dialects.postgresql import insert
+
+from ...config import CONFIG
+from ..typedefs import FILTERFIELD_TYPE, QUERYFIELD_TYPE
+
+
+def _insert_on_conflict_update(table, conn, keys, data_iter, indexes):
+    data = [dict(zip(keys, row)) for row in data_iter]
+    stmt = insert(table.table).values(data)
+    stmt = stmt.on_conflict_do_update(index_elements=indexes, set_={k: getattr(stmt.excluded, k) for k in keys})
+    result = conn.execute(stmt)
+    return result.rowcount
+
+
+def _insert_on_conflict_nothing(table, conn, keys, data_iter):
+    data = [dict(zip(keys, row)) for row in data_iter]
+    stmt = insert(table.table).values(data).on_conflict_do_nothing(index_elements=keys)
+    result = conn.execute(stmt)
+    return result.rowcount
+
+
+class SqlManager:
+    """
+    This class is used to manage SQL operations.
+
+    :ivar sqlalchemy.engine.Engine engine: An instance of the SQLAlchemy Engine class for executing SQL operations.
+    """
+
+    def __init__(self):
+        self.engine = create_engine(CONFIG["sqlconnstr"])
+
+    def _execute(self, sql_executable: Union[str, sql.base.Executable]) -> Any:
+        if isinstance(sql_executable, str):
+            sql_executable = text(sql_executable)
+        with self.engine.begin() as conn:
+            return conn.execute(sql_executable)
+
+    def add_index(self, table_name: str, columns: Union[str, list[str]], unique: bool = True):
+        """
+        Adds an index to a table.
+
+        :param table_name: The name of the table to add the index to.
+        :type table_name: str
+        :param columns: The column(s) to include in the index. It can be a single column name or a list of column names.
+        :type columns: Union[str, list[str]]
+        :param unique: Whether the index should enforce unique values. Defaults to True.
+        :type unique: bool
+        """
+        if isinstance(columns, str):
+            columns = [columns]
+
+        index_name = f"uix_{table_name}_{'_'.join(columns)}"
+        table_meta = MetaData()
+        table = Table(table_name, table_meta, autoload_with=self.engine)
+        columns = [getattr(table.c, colname) for colname in columns if colname in table.c.keys()]
+        index = Index(index_name, *columns, unique=unique)
+        index.create(bind=self.engine)
+
+    def insert(
+        self,
+        table_name: str,
+        df: pd.DataFrame,
+        upsert: bool = True,
+        other_unique_index_columns: Sequence[str] = (),
+        other_non_unique_index_columns: Sequence[str] = (),
+    ):
+        """
+        Inserts data into a table.
+
+        :param table_name: The name of the table to insert data into.
+        :type table_name: str
+        :param df: The data to insert. It should be a DataFrame where the column names match the table columns.
+        :type df: pd.DataFrame
+        :param upsert: Whether to update the table if the data already exists. Defaults to True.
+        :type upsert: bool
+        :param other_unique_index_columns: Other columns to enforce unique values on. Defaults to an empty sequence.
+        :type other_unique_index_columns: Sequence[str]
+        :param other_non_unique_index_columns: Other columns to add non-unique indexes to. Defaults to an empty sequence.
+        :type other_non_unique_index_columns: Sequence[str]
+        :return: The number of rows inserted.
+        :rtype: int
+        """
+        if_exists: Literal["replace", "append"] = "append"
+        inspector = inspect(self.engine)
+        new_table = not inspector.has_table(table_name)
+        method = partial(_insert_on_conflict_update, indexes=df.index.names) if upsert and not new_table else None
+        num_rows = df.to_sql(
+            table_name,
+            self.engine,
+            if_exists=if_exists,
+            index=True,
+            index_label=df.index.names,
+            method=method,
+        )
+        if new_table:
+            self.add_index(table_name, df.index.names, unique=True)
+            for column in other_unique_index_columns:
+                self.add_index(table_name, column, unique=True)
+            for column in other_non_unique_index_columns:
+                self.add_index(table_name, column, unique=False)
+        return num_rows
+
+    def _convert_to_sqlalchemy_type(self, column_type: Type, **kwargs):
+        if isinstance(column_type, type):
+            column_type = column_type.__name__
+        if column_type == "str":
+            return String(**kwargs)
+        if column_type == "int":
+            return Integer()
+        if column_type == "float":
+            return REAL()
+        raise ValueError(f"Unsupported column type {column_type}")
+
+    def insert_column(self, table_name: str, column_name: str, column_type: Union[str, Type], type_kwargs: dict = None):
+        """
+        Inserts a new column into a table.
+
+        :param table_name: The name of the table to insert the column into.
+        :type table_name: str
+        :param column_name: The name of the new column.
+        :type column_name: str
+        :param column_type: The data type of the new column. It can be a string or a Python type.
+        :type column_type: Union[str, Type]
+        :param type_kwargs: Additional keyword arguments for the data type. Defaults to None.
+        :type type_kwargs: dict, optional
+        """
+        type_kwargs = type_kwargs or {}
+        sql_code = f"ALTER TABLE {table_name} ADD COLUMN {column_name} {str(self._convert_to_sqlalchemy_type(column_type, **type_kwargs))}"
+        self._execute(sql_code)
+
+    def delete_column(self, table_name: str, column_name: str):
+        """
+        Deletes a column from a table.
+
+        :param table_name: The name of the table to delete the column from.
+        :type table_name: str
+        :param column_name: The name of the column to delete.
+        :type column_name: str
+        """
+        self._execute(f"ALTER TABLE {table_name} DROP COLUMN {column_name}")
+
+    def rename_column(self, table_name: str, old_column_name: str, new_column_name: str):
+        """
+        Renames a column in a table.
+
+        :param table_name: The name of the table containing the column to rename.
+        :type table_name: str
+        :param old_column_name: The current name of the column.
+        :type old_column_name: str
+        :param new_column_name: The new name for the column.
+        :type new_column_name: str
+        """
+        # check if any index is referring to the column
+        sql_code = f"SELECT indexname, indexdef FROM pg_indexes WHERE indexdef LIKE '%%(%%{old_column_name}%%)%%' and tablename = '{table_name}'"
+        index_refering_column = dict(self.engine.execute(sql_code).fetchall())
+
+        # drop indexes referring to the column
+        for index_name in index_refering_column:
+            self._execute(f"DROP INDEX IF EXISTS {index_name}")
+
+        # rename the column
+        self._execute(f"ALTER TABLE {table_name} RENAME COLUMN {old_column_name} TO {new_column_name}")
+
+        # recreate the indexes
+        for index_name, index_def in index_refering_column.items():
+            new_index_name = index_name.replace(old_column_name, new_column_name)
+            new_index_def = re.sub(rf"\(([^)]*?){old_column_name}([^)]*?)\)", rf"(\1{new_column_name}\2)", index_def)
+            new_index_def = new_index_def.replace(index_name, new_index_name)
+            self._execute(new_index_def)
+
+    def read_range_data(
+        self,
+        table_name: str,
+        query_fields="*",
+        start_time: pd.Timestamp = None,
+        end_time: pd.Timestamp = None,
+        filter_fields=None,
+    ):
+        meta = MetaData()
+        table = Table(table_name, meta, autoload_with=self.engine)
+
+        if query_fields != "*":
+            query_fields = [table.columns[field] for field in query_fields]
+        else:
+            query_fields = table
+
+        stmt = select(*query_fields)
+
+        conditions = []
+        if start_time:
+            conditions.append(getattr(table.c, "timestamp", table.c.end_time) >= start_time)
+        if end_time:
+            conditions.append(getattr(table.c, "timestamp", table.c.start_time) <= end_time)
+        if filter_fields:
+            conditions.extend(
+                [
+                    (
+                        table.columns[field].in_(filter_values)
+                        if isinstance(filter_values, Container) and not isinstance(filter_values, (str, bytes))
+                        else table.columns[field] == filter_values
+                    )
+                    for field, filter_values in filter_fields.items()
+                ]
+            )
+        stmt = stmt.where(sql.and_(*conditions))
+        res = self._execute(stmt)
+        return pd.DataFrame(res.fetchall(), columns=res.keys())
+
+    def read_data(self, table_name: str, query_fields: QUERYFIELD_TYPE = "*", filter_fields=None):
+        """
+        Reads data from a table.
+
+        :param table_name: The name of the table to read data from.
+        :type table_name: str
+        :param query_fields: The fields to query. By default, it queries all fields. Defaults to "*".
+        :type query_fields: QUERYFIELD_TYPE, optional
+        :param filter_fields: Additional fields to filter by. The keys are the field names and the values are the filter values. Defaults to None.
+        :type filter_fields: dict, optional
+        :return: A DataFrame containing the queried data.
+        :rtype: pd.DataFrame
+        """
+
+        meta = MetaData()
+        table = Table(table_name, meta, autoload_with=self.engine)
+
+        if query_fields != "*":
+            query_fields = [table.columns[field] for field in query_fields]
+            stmt = select(*query_fields)
+        else:
+            query_fields = table
+            stmt = select(table)
+
+        if filter_fields:
+            conditions = [
+                (
+                    table.columns[field].in_(filter_values)
+                    if isinstance(filter_values, Container) and not isinstance(filter_values, (str, bytes))
+                    else table.columns[field] == filter_values
+                )
+                for field, filter_values in filter_fields.items()
+            ]
+            stmt = stmt.where(sql.and_(*conditions))
+
+        # cannot use pandas.read_sql here as it discards timezone info
+        res = self._execute(stmt)
+        pd.read_sql_query()
+        return pd.DataFrame(res.fetchall(), columns=res.keys())
+
+    def read_data_across_tables(
+        self,
+        table_names: Sequence[str],
+        joined_columns: Sequence[str],
+        query_fields: QUERYFIELD_TYPE = "*",
+        filter_fields: FILTERFIELD_TYPE = None,
+    ):
+        """
+        Reads data from multiple tables.
+
+        :param table_names: The names of the tables to read data from.
+        :type table_names: Sequence[str]
+        :param joined_columns: The columns to join the tables on.
+        :type joined_columns: Sequence[str]
+        :param query_fields: The fields to query. By default, it queries all fields. Defaults to "*".
+        :type query_fields: QUERYFIELD_TYPE, optional
+        :param filter_fields: Additional fields to filter by. The keys are the field names and the values are the filter values. Defaults to None.
+        :type filter_fields: FILTERFIELD_TYPE, optional
+        :return: A DataFrame containing the queried data.
+        :rtype: pd.DataFrame
+        """
+        meta = MetaData()
+        tables = {table_name: Table(table_name, meta, autoload_with=self.engine) for table_name in table_names}
+
+        joined_table = reduce(
+            lambda x, y: x.join(y, sql.and_(*[x.columns[col] == y.columns[col] for col in joined_columns])),
+            tables.values(),
+        )
+
+        query_fields_rel = []
+        if query_fields != "*":
+            for table_name, colnames in query_fields.items():
+                if isinstance(colnames, str):
+                    query_fields_rel.append(tables[table_name].columns[colnames])
+                else:
+                    query_fields_rel.extend([tables[table_name].columns[colname] for colname in colnames])
+        else:
+            query_fields_rel = [text("*")]
+
+        stmt = select(*query_fields_rel).select_from(joined_table)
+
+        if filter_fields:
+            conditions = []
+            for table_name, colnames in filter_fields.items():
+                table = tables[table_name]
+                for field, filter_values in colnames.items():
+                    conditions.append(
+                        table.columns[field].in_(filter_values)
+                        if isinstance(filter_values, Container) and not isinstance(filter_values, (str, bytes))
+                        else table.columns[field] == filter_values
+                    )
+
+            stmt = stmt.where(sql.and_(*conditions))
+
+        # cannot use pandas.read_sql here as it discards timezone info
+        res = self._execute(stmt)
+        return pd.DataFrame(res.fetchall(), columns=res.keys())

trade_database_manager-0.0.1.dev4/trade_database_manager/core/typedefs.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/22 17:48
+# @Author  : YQ Tsui
+# @File    : typedefs.py
+# @Purpose :
+
+from typing import Dict, Literal, Optional, Sequence, TypeVar, Union
+
+T = TypeVar("T")
+T_SeqT = Union[T, Sequence[T]]
+Opt_T_SeqT = Optional[Union[T, Sequence[T]]]
+
+T_DictT = Union[T, Dict[str, T]]
+Opt_T_DictT = Optional[Union[T, Dict[str, T]]]
+
+QUERYFIELD_TYPE = T_DictT[Union[Literal["*"], Sequence[str]]]
+FILTERFIELD_TYPE = Opt_T_DictT[Dict[str, Union[str, Sequence[str], int, float, bool]]]

trade_database_manager-0.0.1.dev4/trade_database_manager/manager/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/21 12:41
+# @Author  : YQ Tsui
+# @File    : __init__.py
+# @Purpose :
+
+from .metadata_sql import MetadataSql
+
+__all__ = ("MetadataSql",)

trade_database_manager-0.0.1.dev4/trade_database_manager/manager/fields_data_type.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/19 16:44
+# @Author  : YQ Tsui
+# @File    : fields_data_type.py
+# @Purpose :
+
+FIELD_DATA_TYPE_SQL = {
+    "ticker": "VARCHAR(20)",
+    "name": "VARCHAR(20)",
+    "currency": "VARCHAR(6)",
+    "exchange": "VARCHAR(10)",
+    "timezone": "VARCHAR(30)",
+    "tick_size": "REAL",
+    "lot_size": "REAL",
+    "min_lots": "REAL",
+    "market_tplus": "INTEGER",
+    "listed_date": "DATE",
+    "delisted_date": "DATE",
+    "sector": "VARCHAR(30)",
+    "industry": "VARCHAR(36)",
+    "country": "VARCHAR(36)",
+    "board_type": "VARCHAR(20)",
+}

trade_database_manager-0.0.1.dev4/trade_database_manager/manager/metadata_sql.py

@@ -0,0 +1,220 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/15 20:28
+# @Author  : YQ Tsui
+# @File    : metadata_sql.py
+# @Purpose : Instrument metadata stored in SQL database
+
+from collections.abc import Container
+from typing import Union, cast
+
+import pandas as pd
+
+from ..core.sql.sqlmanager import SqlManager
+from .typedefs import EXCHANGE_LITERALS, INST_TYPE_LITERALS, Opt_T_SeqT, T_DictT
+
+COMMON_METADATA_COLUMNS = [
+    "name",
+    "trading_code",
+    "inst_type",
+    "currency",
+    "timezone",
+    "tick_size",
+    "lot_size",
+    "min_lots",
+    "market_tplus",
+    "listed_date",
+    "delisted_date",
+]
+TYPE_METADATA_COLUMNS = {
+    "STK": ["sector", "industry", "country", "state", "board_type", "issue_price"],
+}
+
+
+class MetadataSql:
+    """
+    This class is used to manage instrument metadata stored in an SQL database.
+
+    This is a singleton class. Just call MetadataSql() to get the instance.
+    """
+
+    _instance = None
+    _manager = None
+
+    def __new__(cls):
+        if not isinstance(cls._instance, cls):
+            cls._instance = super(MetadataSql, cls).__new__(cls)
+            cls._manager = SqlManager()
+        return cls._instance
+
+    def update_instrument_metadata(self, data: Union[pd.DataFrame, list[dict], dict]):
+        """
+        Updates the instrument metadata in the database.
+
+        :param data: The data to be updated. It can be a DataFrame, a list of dictionaries, or a single dictionary.
+        :type data: Union[pd.DataFrame, list[dict], dict]
+        """
+        type_specific_columns = set(data.columns) - set(COMMON_METADATA_COLUMNS)
+        if "inst_type" not in data.columns and bool(type_specific_columns):
+            raise ValueError(
+                f"Non-common columns found ({','.join(type_specific_columns)}) but inst_type column not provided."
+            )
+        if isinstance(data, dict):
+            data = [data]
+        if isinstance(data, list):
+            data = pd.DataFrame(data)
+        if "ticker" in data.columns and "exchange" in data.columns:
+            data.set_index(["ticker", "exchange"], inplace=True)
+        else:
+            assert set(data.index.names) == {"ticker", "exchange"}, "Index names must be 'ticker' and 'exchange'."
+        data_common = data[data.columns.intersection(COMMON_METADATA_COLUMNS)]
+
+        self._manager.insert("instruments", data_common, upsert=True)
+        if "inst_type" in data.columns:
+            for inst_type, columns in TYPE_METADATA_COLUMNS.items():
+                data_type_df = data.loc[data.inst_type == inst_type, data.columns.intersection(columns)]
+                if not data_type_df.empty:
+                    self._manager.insert(f"instruments_{inst_type.lower()}", data_type_df, upsert=True)
+
+    def _convert_datetime_columns(self, data: pd.DataFrame):
+        for col in ["listed_date", "delisted_date"]:
+            if col in data.columns:
+                data[col] = data[col].apply(pd.to_datetime)
+
+    def read_metadata(
+        self,
+        ticker: Opt_T_SeqT[str] = None,
+        exchange: Opt_T_SeqT[EXCHANGE_LITERALS] = None,
+        query_fields="*",
+        filter_fields=None,
+    ) -> T_DictT[pd.DataFrame]:
+        """
+        Reads metadata from the database based on the provided filters.
+
+        :param ticker: The ticker(s) to filter by. It can be a single ticker or a sequence of tickers. Defaults to None.
+        :type ticker: Opt_T_SeqT[str], optional
+        :param exchange: The exchange(s) to filter by. It can be a single exchange or a sequence of exchanges. Defaults to None.
+        :type exchange: Opt_T_SeqT[EXCHANGE_LITERALS], optional
+        :param query_fields: The fields to query. By default, it queries all fields. Defaults to "*".
+        :type query_fields: str, optional
+        :param filter_fields: Additional fields to filter by. The keys are the field names and the values are the filter values. Defaults to None.
+        :type filter_fields: dict, optional
+        :return: A dictionary of DataFrames containing the queried metadata.
+        :rtype: T_DictT[pd.DataFrame]
+        """
+        filter_fields = filter_fields or {}
+        if ticker is not None:
+            filter_fields["ticker"] = ticker
+        if exchange is not None:
+            if not isinstance(exchange, str) and isinstance(exchange, Container) and ticker is not None:
+                assert len(exchange) == len(ticker), "Exchange must be a single value or the same length as ticker."
+            filter_fields["exchange"] = exchange
+        query_fields_common = (
+            ["ticker", "exchange"] + [f for f in query_fields if f in COMMON_METADATA_COLUMNS]
+            if query_fields != "*"
+            else "*"
+        )
+        filter_fields_common = {
+            k: v for k, v in filter_fields.items() if k in ["ticker", "exchange"] + COMMON_METADATA_COLUMNS
+        }
+        all_fields_common = (
+            len(query_fields_common) == len(query_fields)
+            and len(filter_fields_common) == len(filter_fields)
+            and query_fields != "*"
+        )
+        if not (query_fields == "*" or "inst_type" in query_fields):
+            query_fields_common.append("inst_type")
+        common_df = self._manager.read_data(
+            "instruments", query_fields=query_fields_common, filter_fields=filter_fields
+        )
+        self._convert_datetime_columns(common_df)
+        if common_df.empty:
+            return {}
+        res = {}
+        for inst_type, common_df_by_type in common_df.groupby("inst_type"):
+            inst_type = cast(INST_TYPE_LITERALS, inst_type)
+            if all_fields_common:
+                res[inst_type] = common_df_by_type
+                continue
+            query_fields_type = (
+                ["ticker", "exchange"] + [f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]]
+                if query_fields != "*"
+                else "*"
+            )
+            filter_fields_type = {k: v for k, v in filter_fields.items() if k in TYPE_METADATA_COLUMNS[inst_type]}
+            filter_fields_type["ticker"] = common_df_by_type["ticker"].to_list()
+            filter_fields_type["exchange"] = common_df_by_type["exchange"].to_list()
+            type_df = self._manager.read_data(
+                f"instruments_{inst_type.lower()}", query_fields=query_fields_type, filter_fields=filter_fields_type
+            )
+            type_df = common_df_by_type.merge(type_df, on=["ticker", "exchange"], how="inner")
+            res[inst_type] = type_df.set_index(["ticker", "exchange"])
+        return res
+
+    def read_metadata_for_insttype(
+        self,
+        inst_type: INST_TYPE_LITERALS,
+        ticker: Opt_T_SeqT[str] = None,
+        exchange: Opt_T_SeqT[EXCHANGE_LITERALS] = None,
+        query_fields="*",
+        filter_fields=None,
+    ) -> pd.DataFrame:
+        """
+        Reads metadata for a specific instrument type from the database based on the provided filters.
+
+        :param inst_type: The instrument type to filter by.
+        :type inst_type: INST_TYPE_LITERALS
+        :param ticker: The ticker(s) to filter by. It can be a single ticker or a sequence of tickers. Defaults to None.
+        :type ticker: Opt_T_SeqT[str], optional
+        :param exchange: The exchange(s) to filter by. It can be a single exchange or a sequence of exchanges. Defaults to None.
+        :type exchange: Opt_T_SeqT[EXCHANGE_LITERALS], optional
+        :param query_fields: The fields to query. By default, it queries all fields. Defaults to "*".
+        :type query_fields: str, optional
+        :param filter_fields: Additional fields to filter by. The keys are the field names and the values are the filter values. Defaults to None.
+        :type filter_fields: dict, optional
+        :return: A DataFrame containing the queried metadata for the specified instrument type.
+        :rtype: pd.DataFrame
+        """
+        filter_fields = filter_fields or {}
+        filter_fields["inst_type"] = inst_type
+        if ticker is not None:
+            filter_fields["ticker"] = ticker
+        if exchange is not None:
+            if not isinstance(exchange, str) and isinstance(exchange, Container) and ticker is not None:
+                assert len(exchange) == len(ticker), "Exchange must be a single value or the same length as ticker."
+            filter_fields["exchange"] = exchange
+
+        if query_fields == "*":
+            query_fields_cross = "*"
+        else:
+            query_fields_common = ["ticker", "exchange"] + [f for f in query_fields if f in COMMON_METADATA_COLUMNS]
+            query_fields_type = ["ticker", "exchange"] + [
+                f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]
+            ]
+            query_fields_cross = {
+                "instruments": query_fields_common,
+                f"instruments_{inst_type.lower()}": query_fields_type,
+            }
+
+        filter_fields_common = {
+            k: v for k, v in filter_fields.items() if k in ["ticker", "exchange"] + COMMON_METADATA_COLUMNS
+        }
+        filter_fields_type = {
+            k: v for k, v in filter_fields.items() if k in ["ticker", "exchange"] + TYPE_METADATA_COLUMNS[inst_type]
+        }
+        filter_fields_cross = {
+            "instruments": filter_fields_common,
+            f"instruments_{inst_type.lower()}": filter_fields_type,
+        }
+        df = self._manager.read_data_across_tables(
+            ["instruments", f"instruments_{inst_type.lower()}"],
+            joined_columns=["ticker", "exchange"],
+            query_fields=query_fields_cross,
+            filter_fields=filter_fields_cross,
+        )
+
+        if isinstance(df.columns, pd.Index):
+            df = df.loc[:, ~df.columns.duplicated()].set_index(["ticker", "exchange"])
+        else:
+            df = df.set_index(["ticker", "exchange"])
+        self._convert_datetime_columns(df)
+        return df

trade_database_manager-0.0.1.dev4/trade_database_manager/manager/typedefs.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/22 16:18
+# @Author  : YQ Tsui
+# @File    : typedefs.py
+# @Purpose : Type hints for convenience
+
+from typing import Dict, Literal, Optional, Sequence, TypeVar, Union
+
+INST_TYPE_LITERALS = Literal["STK", "FUT", "OPT", "IDX", "ETF", "FND", "BND", "CASH", "CRYPTO"]
+EXCHANGE_LITERALS = Literal[
+    "SSE",
+    "SZSE",
+    "HKEX",
+    "CFFEX",
+    "SHFE",
+    "DCE",
+    "CZCE",
+    "SGX",
+    "CBOT",
+    "CME",
+    "COMEX",
+    "NYMEX",
+    "ICE",
+    "LME",
+    "TOCOM",
+    "JPX",
+    "KRX",
+    "ASX",
+    "NSE",
+    "BSE",
+    "NSE",
+    "BSE",
+    "MCX",
+    "MOEX",
+    "TSE",
+    "TWSE",
+    "SET",
+    "IDX",
+    "CRYPTO",
+    "SMART",
+]
+
+T = TypeVar("T")
+T_SeqT = Union[T, Sequence[T]]
+Opt_T_SeqT = Optional[Union[T, Sequence[T]]]
+
+T_DictT = Union[T, Dict[str, T]]
+Opt_T_DictT = Optional[Union[T, Dict[str, T]]]

{trade_database_manager-0.0.1.dev1 → trade_database_manager-0.0.1.dev4/trade_database_manager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.1.dev1
+Version: 0.0.1.dev4
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux
@@ -52,5 +52,26 @@ host    all            tradedbadmin     ::/0                    scram-sha-256
 
 ### KDB+ Initialization
 
+#### get license
+Request a 64-bit license from [Kx Systems](https://kx.com/kdb-insights-personal-edition-license-download). Fill the form
+then the system will send an email with the license file and a base64 key (Either can be used to license the product).
 
-```q
+#### Server side (Linux system)
+1. Put the executable `q` and license in desired folder, say `/opt/l64` directory.
+2. Create a folder to store the data, say `/opt/data` directory.
+3. Set the environment variable `QLIC` to the directory where the license file is stored. And `QHOME` to the data directory.
+4. Run the `q` executable with the following command:
+    ```bash
+    $ q -p 5000 -s 1 -w 100 -t 1000 -T 1000 -U /opt/l64/trade.q
+    ```
+   - `-p 5000`: This sets the port number for the kdb+ process. In this case, the port number is 5000.
+
+   - `-s 1`: This sets the number of secondary threads. In this case, it's set to 1.
+
+   - `-w 100`: This sets the workspace heap size. In this case, it's set to 100 MB.
+
+   - `-t 1000`: This sets the timer interval in milliseconds. In this case, it's set to 1000 milliseconds, or 1 second.
+
+   - `-T 1000`: This sets the timeout in seconds for client queries. In this case, it's set to 1000 seconds.
+
+   - `-U /opt/l64/trade.q`: This sets the access control list file. In this case, the file is located at `/opt/l64/trade.q`. This file contains a list of usernames and passwords for clients that are allowed to connect to the kdb+ process.

{trade_database_manager-0.0.1.dev1 → trade_database_manager-0.0.1.dev4}/trade_database_manager.egg-info/SOURCES.txt

@@ -10,10 +10,13 @@ trade_database_manager.egg-info/dependency_links.txt
 trade_database_manager.egg-info/requires.txt
 trade_database_manager.egg-info/top_level.txt
 trade_database_manager/core/__init__.py
+trade_database_manager/core/typedefs.py
 trade_database_manager/core/kdb/__init__.py
 trade_database_manager/core/sql/__init__.py
 trade_database_manager/core/sql/sqlmanager.py
 trade_database_manager/core/sql/sqlreader.py
 trade_database_manager/core/sql/sqlwriter.py
-trade_database_manager/tools/__init__.py
-trade_database_manager/tools/init_config.py
+trade_database_manager/manager/__init__.py
+trade_database_manager/manager/fields_data_type.py
+trade_database_manager/manager/metadata_sql.py
+trade_database_manager/manager/typedefs.py

trade_database_manager-0.0.1.dev1/README.md

@@ -1,34 +0,0 @@
-<table border=1 cellpadding=10>
-<tr>
-<td style="color: red;">
-
-#### \*\*\* IMPORTANT NOTICE \*\*\*
-
-<p style="color: red">This package is <b>not</b> in a usable stage. It is only uploaded for convenience of developing and testing.</p>
-
-</td></tr></table>
-
-
-
-### Postgreslq Initialization
-
-```sql
-CREATE DATABASE trade_data;
-CREATE USER tradedbadmin WITH PASSWORD 'trade_password';
-GRANT ALL PRIVILEGES ON DATABASE trade_data TO tradedbadmin;
-\connect trade_data;
-GRANT ALL ON SCHEMA public TO tradedbadmin;
-```
-
-Add the following lines in the `pg_hba.conf` file to allow password authentication for the `tradedbadmin` role.
-
-```
-# trade database
-host    all            tradedbadmin     0.0.0.0/0               scram-sha-256
-host    all            tradedbadmin     ::/0                    scram-sha-256
-```
-
-### KDB+ Initialization
-
-
-```q

trade_database_manager-0.0.1.dev1/trade_database_manager/core/sql/sqlmanager.py

@@ -1,102 +0,0 @@
-from functools import partial
-from typing import Literal, Sequence, Union
-
-import pandas as pd
-from sqlalchemy import create_engine, inspect
-from sqlalchemy.dialects.postgresql import insert
-import re
-from ...config import CONFIG
-
-
-def _insert_on_conflict_update(table, conn, keys, data_iter, indexes):
-    data = [dict(zip(keys, row)) for row in data_iter]
-    stmt = insert(table.table).values(data)
-    stmt = stmt.on_conflict_do_update(index_elements=indexes, set_={k: getattr(stmt.excluded, k) for k in keys})
-    result = conn.execute(stmt)
-    return result.rowcount
-
-
-def _insert_on_conflict_nothing(table, conn, keys, data_iter):
-    data = [dict(zip(keys, row)) for row in data_iter]
-    stmt = insert(table.table).values(data).on_conflict_do_nothing(index_elements=keys)
-    result = conn.execute(stmt)
-    return result.rowcount
-
-
-class SqlManager:
-    def __init__(
-        self,
-    ):
-        self.engine = create_engine(CONFIG["sqlconnstr"])
-
-    def add_index(self, table_name: str, columns: Union[str, Sequence[str]], unique: bool = True):
-        if isinstance(columns, str):
-            columns = [columns]
-        index_name = f"uix_{table_name}_{'_'.join(columns)}"
-        columns_str = ", ".join(columns)
-        unique_str = "UNIQUE" if unique else ""
-        self.engine.execute(f"CREATE {unique_str} INDEX {index_name} ON {table_name} ({columns_str})")
-
-    def insert(
-        self,
-        table_name: str,
-        df: pd.DataFrame,
-        upsert: bool = True,
-        other_unique_index_columns: Sequence[str] = (),
-        other_non_unique_index_columns: Sequence[str] = (),
-    ):
-        if_exists: Literal["replace", "append"] = "append"
-        inspector = inspect(self.engine)
-        new_table = not inspector.has_table(table_name)
-        method = (
-            partial(_insert_on_conflict_update, indexes=df.index.names)
-            if upsert and not new_table
-            else None
-        )
-        num_rows = df.to_sql(
-            table_name,
-            self.engine,
-            if_exists=if_exists,
-            index=True,
-            index_label=df.index.names,
-            method=method,
-        )
-        if new_table:
-            self.add_index(table_name, df.index.names)
-            for column in other_unique_index_columns:
-                self.add_index(table_name, column, unique=True)
-            for column in other_non_unique_index_columns:
-                self.add_index(table_name, column, unique=False)
-        return num_rows
-
-    def rename_column(self, table_name: str, old_column_name: str, new_column_name: str):
-        # check if any index is referring to the column
-        sql_code = f"SELECT indexname, indexdef FROM pg_indexes WHERE indexdef LIKE '%%(%%{old_column_name}%%)%%' and tablename = '{table_name}'"
-        index_refering_column = dict(self.engine.execute(sql_code).fetchall())
-
-        # drop indexes referring to the column
-        for index_name in index_refering_column:
-            self.engine.execute(f"DROP INDEX IF EXISTS {index_name}")
-
-        # rename the column
-        self.engine.execute(f"ALTER TABLE {table_name} RENAME COLUMN {old_column_name} TO {new_column_name}")
-
-        # recreate the indexes
-        for index_name, index_def in index_refering_column.items():
-            new_index_name = index_name.replace(old_column_name, new_column_name)
-            new_index_def = re.sub(rf"\(([^)]*?){old_column_name}([^)]*?)\)", rf"(\1{new_column_name}\2)", index_def)
-            new_index_def = new_index_def.replace(index_name, new_index_name)
-            self.engine.execute(new_index_def)
-
-    def read_range_data(
-        self, table_name: str, query_fields, start_time: pd.Timestamp, end_time: pd.Timestamp, filter_fields=None
-    ):
-        sql_code = f"SELECT {', '.join(query_fields)} FROM {table_name} WHERE end_time >= '{start_time}' AND start_time <= '{end_time}'"
-        if filter_fields:
-            sql_code += (
-                f" AND {' AND '.join([f'{field} IN {tuple(filter_values)}' for field, filter_values in filter_fields])}"
-            )
-
-        # cannot use pandas.read_sql here as it discards timezone info
-        res = self.engine.execute(sql_code)
-        return pd.DataFrame(res.fetchall(), columns=res.keys())

trade_database_manager-0.0.1.dev1/trade_database_manager/tools/init_config.py

@@ -1,42 +0,0 @@
-import argparse
-import importlib.util
-from pathlib import Path
-
-from ruamel.yaml import YAML
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-u", "--username", type=str)
-    parser.add_argument("-pw", "--password", type=str)
-    parser.add_argument(
-        "--host",
-        type=str,
-        default="localhost",
-        help="host of both kdb and sql, will be overwritten if sqlhost or kdbhost is specified",
-    )
-    parser.add_argument("--sqlhost", type=str, default="", help="host of sql")
-    parser.add_argument("--sqldb", type=str, default="trade_data", help="database name of sql")
-    parser.add_argument("--kdbhost", type=str, default="", help="host of kdb")
-    parser.add_argument("--sqlport", type=str, default="5432", help="port of sql")
-    parser.add_argument("--kdbport", type=str, default="5000", help="port of kdb")
-
-    args = parser.parse_args()
-
-    kdb_host = args.host if args.kdbhost == "" else args.kdbhost
-    sql_host = args.host if args.sqlhost == "" else args.sqlhost
-
-    sql_protocol = "postgresql" if importlib.util.find_spec("psycopg2") is None else "postgresql+psycopg2"
-
-    config = {
-        "username": args.username,
-        "password": args.password,
-        "kdbhost": kdb_host,
-        "kdbport": args.kdbport,
-        "sqlconnstr": f"{sql_protocol}://{args.username}:{args.password}@{sql_host}:{args.sqlport}/{args.sqldb}",
-    }
-
-    yaml = YAML(typ="safe")
-    config_folder = Path("~/.tradedbmgr").expanduser()
-    config_folder.mkdir(exist_ok=True)
-    with open(config_folder.joinpath("config.yaml"), "w") as f:
-        yaml.dump(config, f)