trade-database-manager 0.0.1.dev4__tar.gz → 0.0.3.dev1__tar.gz

{trade_database_manager-0.0.1.dev4/trade_database_manager.egg-info → trade_database_manager-0.0.3.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.1.dev4
+Version: 0.0.3.dev1
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux

{trade_database_manager-0.0.1.dev4 → trade_database_manager-0.0.3.dev1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trade_database_manager"
-version = "0.0.1.dev4"
+version = "0.0.3.dev1"
 description = "A wrapper of kdb and sql for convenient trade data management."
 readme = "README.md"
 authors = [

trade_database_manager-0.0.3.dev1/trade_database_manager/core/kdb/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/15 20:28
+# @Author  : YQ Tsui
+# @File    : __init__.py
+# @Purpose :

trade_database_manager-0.0.3.dev1/trade_database_manager/core/kdb/kdbmanager.py

@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/27 12:26
+# @Author  : YQ Tsui
+# @File    : kdbmanager.py
+# @Purpose :
+import os.path
+import pandas as pd
+import pykx
+from ...config import CONFIG
+
+
+class KdbManager:
+
+    _instance = None
+
+    @classmethod
+    def instance(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+            cls._instance.__init__()
+        return cls._instance
+
+    def __init__(self):
+        self.host = CONFIG["kdbhost"]
+        self.port = CONFIG["kdbport"]
+        # self.username = CONFIG["username"]
+        # self.password = CONFIG["password"]
+        self.username = ""
+        self.password = ""
+        # print(self.host, self.port, self.username, self.password)
+
+    def path_exists(self, path: str):
+        """
+        Checks if a path exists in the kdb database.
+
+        :param path: The path to check.
+        :type path: str
+        """
+        with pykx.QConnection(self.host, self.port, username=self.username, password=self.password) as conn:
+            return bool(conn(f".path.exists `:{path}"))
+
+    def create_folder(self, path: str):
+        """
+        Creates a folder in the kdb database.
+
+        :param path: The path of the folder.
+        :type path: str
+        """
+        with pykx.QConnection(self.host, self.port, username=self.username, password=self.password) as conn:
+            conn(f".path.mkdir \"{path}\"")
+
+    def write(self, table_name: str, data: pd.DataFrame, path: str = "", splayed: bool = False):
+        """
+        Writes data to the kdb database.
+
+        :param table_name: The table name in the kdb database.
+        :type table_name: str
+        :param data: The data to be written.
+        :type data: pd.DataFrame
+        :param path: The path of data
+        :type path: str
+        :param splayed: Whether the table is splayed.
+        :type splayed: bool
+        """
+
+        real_path = os.path.join(path, table_name) if path else table_name
+        if splayed:
+            real_path += "/"
+        with pykx.QConnection(self.host, self.port, username=self.username, password=self.password) as conn:
+            conn(f"{{`:{real_path}/ set x}}", data)
+
+    def write_partitioned(self, table_name: str, data: pd.DataFrame, path: str = "", partition_func=None, key_column=None):
+        """
+        Writes data to the kdb database with partitioning.
+
+        :param table_name: The table name in the kdb database.
+        :type table_name: str
+        :param data: The data to be written.
+        :type data: pd.DataFrame
+        :param path: The path of data
+        :type path: str
+        :param partition_func: calculate the partition bucket from datetime column
+        :type partition_func: callable
+        :param key_column: Column which is sorted to server as key.
+        :type key_column: str
+        """
+        if isinstance(data.index, pd.DatetimeIndex):
+            data.reset_index(inplace=True, names=["datetime"])
+        assert "datetime" in data.columns, "datetime column not found"
+        data.sort_values(by="datetime", inplace=True)
+        if key_column is not None and key_column in data.columns:
+            with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
+                for bucket, df in data.groupby(partition_func(data["datetime"])):
+                    conn(f'{{`{table_name} set x; .partable.createOrAppend[`:{path};{bucket};`{key_column};`{table_name}]}}', df.reset_index(drop=True))
+        else:
+            with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
+                for bucket, df in data.groupby(by=partition_func(data["datetime"])):
+                    conn(f'{{`{table_name} set x;.Q.dpt[`:{path};{bucket};`{table_name}]}}', df.reset_index(drop=True))
+
+    def read_partitioned(self, table_name: str, path: str = "", fields=None, start_time=None, end_time=None, partition_func=None,
+                         other_conditions=None):
+        """
+        Reads data from the kdb database with partitioning.
+
+        :param table_name: The table name in the kdb database.
+        :type table_name: str
+        :param path: The path of data
+        :type path: str
+        :param start_time: The start time of the data.
+        :type start_time: pd.Timestamp
+        :param end_time: The end time of the data.
+        :type end_time: pd.Timestamp
+        :param partition_func: calculate the partition bucket from datetime column
+        :type partition_func: callable
+        :param other_conditions: Other conditions for the query.
+        :type other_conditions: str
+        """
+        time_format = "%Y.%m.%dD%H:%M:%S.%f"
+        start_time_str = start_time.strftime(time_format) if start_time is not None else None
+        end_time_str = end_time.strftime(time_format) if end_time is not None else None
+        if isinstance(fields, (bytes, str)):
+            fields = [fields]
+        select_clause = f"select from {table_name}" if fields is None else f"select {','.join(fields)} from {table_name}"
+        where_cond = ""
+        if start_time_str is not None and end_time_str is not None:
+            where_cond += f"datetime within ({start_time_str};{end_time_str})"
+            if partition_func is not None:
+                where_cond = f"int in {' '.join(str(x) for x in range(partition_func(start_time), partition_func(end_time) + 1))}" + ',' + where_cond
+        elif start_time_str is not None:
+            where_cond += f"datetime>={start_time_str}"
+            if partition_func is not None:
+                where_cond = f"int>={partition_func(start_time)}" + ',' + where_cond
+        elif end_time_str is not None:
+            where_cond += f"datetime<={end_time_str}"
+            if partition_func is not None:
+                where_cond = f"int<= {partition_func(end_time)}" + ',' + where_cond
+        if other_conditions is not None:
+            where_cond = other_conditions if where_cond == "" else where_cond + ',' + other_conditions
+        if where_cond:
+            where_cond = " where " + where_cond
+        final_query = select_clause + where_cond
+
+        with pykx.QConnection(host=self.host, port=self.port, username=self.username, password=self.password) as conn:
+            conn("`currpath__ set .path.pwd[]") # save current path to currpath__ as following command will change the path
+            try:
+                conn(f"\\l {path}")  # load the path
+                q_table = conn(final_query)
+                return q_table.pd().set_index("datetime")
+            finally:
+                conn('system "cd ", currpath__')

trade_database_manager-0.0.3.dev1/trade_database_manager/core/sql/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/15 20:28
+# @Author  : YQ Tsui
+# @File    : __init__.py
+# @Purpose :

{trade_database_manager-0.0.1.dev4 → trade_database_manager-0.0.3.dev1}/trade_database_manager/core/sql/sqlmanager.py

@@ -1,10 +1,16 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/4/15 20:28
+# @Author  : YQ Tsui
+# @File    : sqlmanager.py
+# @Purpose : Core SQL operations
+
 import re
 from collections.abc import Container
 from functools import partial, reduce
 from typing import Any, Literal, Sequence, Type, Union
 
 import pandas as pd
-from sqlalchemy import REAL, Index, Integer, MetaData, String, Table, create_engine, inspect, select, sql, text
+from sqlalchemy import DOUBLE_PRECISION, Index, Integer, MetaData, String, Table, create_engine, inspect, select, sql, text
 from sqlalchemy.dialects.postgresql import insert
 
 from ...config import CONFIG
@@ -107,7 +113,8 @@ class SqlManager:
                 self.add_index(table_name, column, unique=False)
         return num_rows
 
-    def _convert_to_sqlalchemy_type(self, column_type: Type, **kwargs):
+    @staticmethod
+    def _convert_to_sqlalchemy_type(column_type: Type, **kwargs):
         if isinstance(column_type, type):
             column_type = column_type.__name__
         if column_type == "str":
@@ -115,10 +122,10 @@ class SqlManager:
         if column_type == "int":
             return Integer()
         if column_type == "float":
-            return REAL()
+            return DOUBLE_PRECISION()
         raise ValueError(f"Unsupported column type {column_type}")
 
-    def insert_column(self, table_name: str, column_name: str, column_type: Union[str, Type], type_kwargs: dict = None):
+    def insert_column(self, table_name: str, column_name: str, column_type: str):
         """
         Inserts a new column into a table.
 
@@ -126,13 +133,10 @@ class SqlManager:
         :type table_name: str
         :param column_name: The name of the new column.
         :type column_name: str
-        :param column_type: The data type of the new column. It can be a string or a Python type.
-        :type column_type: Union[str, Type]
-        :param type_kwargs: Additional keyword arguments for the data type. Defaults to None.
-        :type type_kwargs: dict, optional
+        :param column_type: The data type of the new column. It is a string representation of the data type in PostgreSQL.
+        :type column_type: str
         """
-        type_kwargs = type_kwargs or {}
-        sql_code = f"ALTER TABLE {table_name} ADD COLUMN {column_name} {str(self._convert_to_sqlalchemy_type(column_type, **type_kwargs))}"
+        sql_code = f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}"
         self._execute(sql_code)
 
     def delete_column(self, table_name: str, column_name: str):
@@ -250,7 +254,6 @@ class SqlManager:
 
         # cannot use pandas.read_sql here as it discards timezone info
         res = self._execute(stmt)
-        pd.read_sql_query()
         return pd.DataFrame(res.fetchall(), columns=res.keys())
 
     def read_data_across_tables(
@@ -310,3 +313,38 @@ class SqlManager:
         # cannot use pandas.read_sql here as it discards timezone info
         res = self._execute(stmt)
         return pd.DataFrame(res.fetchall(), columns=res.keys())
+
+    def create_ts_table(self, table_name: str, time_column: str, table_columns: dict[str, str],
+                        time_chunk_interval: str = "1 year", other_chunkers: dict[str, int] = None):
+        """
+        Creates a table with a timestamp column.
+
+        :param table_name: The name of the table to create.
+        :type table_name: str
+        :param time_column: The name of the timestamp column.
+        :type time_column: str
+        :param table_columns: The names of the columns to create with their data types. The keys are the column names and the values are the data types in PostgreSQL.
+        :type table_columns: Dict[str, str]
+        :param time_chunk_interval: The interval to chunk the time column by. Defaults to "1 year".
+        :type time_chunk_interval: str
+        :param other_chunkers: Other columns to chunk by. The keys are the column names and the values are the num of hashes. Defaults to None.
+        :type other_chunkers: Dict[str, int], optional
+        """
+        query_create_sensordata_table  = f"""
+        CREATE TABLE {table_name} (
+        {time_column} TIMESTAMP NOT NULL,
+        {", ".join([f"{column_name} {column_type}{' NOT NULL' if column_name in other_chunkers else ''}" for column_name, column_type in table_columns.items()])}
+        );
+        """
+        query_create_sensordata_hypertable = f"SELECT create_hypertable('{table_name}', by_range('{time_column}', INTERVAL '{time_chunk_interval}'));"
+        if other_chunkers is not None:
+            for column_name, num_hash in other_chunkers.items():
+                query_create_sensordata_hypertable += f"\nSELECT add_dimension('{table_name}', by_hash('{column_name}', {num_hash}));"
+
+        uidx_name = f"uix_{table_name}_{'_'.join(other_chunkers.keys())}_{time_column}"
+        query_unique_index = f"CREATE UNIQUE INDEX {uidx_name} ON {table_name} ({', '.join(other_chunkers.keys())}, {time_column});"
+        with self.engine.connect() as conn:
+            conn.execute(text(query_create_sensordata_table))
+            conn.execute(text(query_create_sensordata_hypertable))
+            conn.execute(text(query_unique_index))
+            conn.commit()

{trade_database_manager-0.0.1.dev4 → trade_database_manager-0.0.3.dev1}/trade_database_manager/manager/metadata_sql.py

@@ -133,7 +133,7 @@ class MetadataSql:
         for inst_type, common_df_by_type in common_df.groupby("inst_type"):
             inst_type = cast(INST_TYPE_LITERALS, inst_type)
             if all_fields_common:
-                res[inst_type] = common_df_by_type
+                res[inst_type] = common_df_by_type.set_index(["ticker", "exchange"]) if len(common_df_by_type.columns) > 2 else common_df_by_type
                 continue
             query_fields_type = (
                 ["ticker", "exchange"] + [f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]]
@@ -147,7 +147,7 @@ class MetadataSql:
                 f"instruments_{inst_type.lower()}", query_fields=query_fields_type, filter_fields=filter_fields_type
             )
             type_df = common_df_by_type.merge(type_df, on=["ticker", "exchange"], how="inner")
-            res[inst_type] = type_df.set_index(["ticker", "exchange"])
+            res[inst_type] = type_df.set_index(["ticker", "exchange"]) if len(type_df.columns) > 2 else type_df
         return res
 
     def read_metadata_for_insttype(
@@ -187,9 +187,7 @@ class MetadataSql:
             query_fields_cross = "*"
         else:
             query_fields_common = ["ticker", "exchange"] + [f for f in query_fields if f in COMMON_METADATA_COLUMNS]
-            query_fields_type = ["ticker", "exchange"] + [
-                f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]
-            ]
+            query_fields_type = [f for f in query_fields if f in TYPE_METADATA_COLUMNS[inst_type]]
             query_fields_cross = {
                 "instruments": query_fields_common,
                 f"instruments_{inst_type.lower()}": query_fields_type,
@@ -213,8 +211,8 @@ class MetadataSql:
         )
 
         if isinstance(df.columns, pd.Index):
-            df = df.loc[:, ~df.columns.duplicated()].set_index(["ticker", "exchange"])
-        else:
-            df = df.set_index(["ticker", "exchange"])
+            df = df.loc[:, ~df.columns.duplicated()]
+        if len(df.columns) > 2:
+            df.set_index(["ticker", "exchange"], inplace=True)
         self._convert_datetime_columns(df)
         return df

{trade_database_manager-0.0.1.dev4 → trade_database_manager-0.0.3.dev1/trade_database_manager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.1.dev4
+Version: 0.0.3.dev1
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux

{trade_database_manager-0.0.1.dev4 → trade_database_manager-0.0.3.dev1}/trade_database_manager.egg-info/SOURCES.txt

@@ -1,7 +1,6 @@
 LICENSE
 README.md
 pyproject.toml
-setup.py
 trade_database_manager/__init__.py
 trade_database_manager/config.py
 trade_database_manager.egg-info/PKG-INFO
@@ -12,6 +11,7 @@ trade_database_manager.egg-info/top_level.txt
 trade_database_manager/core/__init__.py
 trade_database_manager/core/typedefs.py
 trade_database_manager/core/kdb/__init__.py
+trade_database_manager/core/kdb/kdbmanager.py
 trade_database_manager/core/sql/__init__.py
 trade_database_manager/core/sql/sqlmanager.py
 trade_database_manager/core/sql/sqlreader.py

trade_database_manager-0.0.1.dev4/setup.py

@@ -1,3 +0,0 @@
-from setuptools import setup
-
-setup()