trade-database-manager 0.0.1.dev0__tar.gz → 0.0.1.dev1__tar.gz

{trade_database_manager-0.0.1.dev0/trade_database_manager.egg-info → trade_database_manager-0.0.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.1.dev0
+Version: 0.0.1.dev1
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux
@@ -11,9 +11,10 @@ Classifier: License :: OSI Approved :: GNU Affero General Public License v3
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pandas
+Requires-Dist: pandas>=2.0.0
 Requires-Dist: numpy
 Requires-Dist: ruamel.yaml
 Requires-Dist: sqlalchemy
@@ -49,3 +50,7 @@ host    all            tradedbadmin     0.0.0.0/0               scram-sha-256
 host    all            tradedbadmin     ::/0                    scram-sha-256
 ```
 
+### KDB+ Initialization
+
+
+```q

{trade_database_manager-0.0.1.dev0 → trade_database_manager-0.0.1.dev1}/README.md

@@ -28,3 +28,7 @@ host    all            tradedbadmin     0.0.0.0/0               scram-sha-256
 host    all            tradedbadmin     ::/0                    scram-sha-256
 ```
 
+### KDB+ Initialization
+
+
+```q

{trade_database_manager-0.0.1.dev0 → trade_database_manager-0.0.1.dev1}/pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "trade_database_manager"
-version = "0.0.1.dev0"
+version = "0.0.1.dev1"
 description = "A wrapper of kdb and sql for convenient trade data management."
 readme = "README.md"
 authors = [
     {name = "Y.Q. Cui", email = "qianyun210603@hotmail.com"},
 ]
-dependencies = ["pandas", "numpy", "ruamel.yaml", "sqlalchemy", "psycopg2"]
+dependencies = ["pandas>=2.0.0", "numpy", "ruamel.yaml", "sqlalchemy", "psycopg2"]
 classifiers = [
     "Operating System :: POSIX :: Linux",
     "Operating System :: Microsoft :: Windows",
@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
 ]
 
 [tool.setuptools.packages.find]
@@ -37,7 +38,7 @@ max-line-length = "120"
 
 [tool.black]
 line-length = 120
-target-version = ['py39']
+target-version = ['py310']
 
 [tool.isort]
 profile = "black"

trade_database_manager-0.0.1.dev1/trade_database_manager/core/sql/sqlmanager.py

@@ -0,0 +1,102 @@
+from functools import partial
+from typing import Literal, Sequence, Union
+
+import pandas as pd
+from sqlalchemy import create_engine, inspect
+from sqlalchemy.dialects.postgresql import insert
+import re
+from ...config import CONFIG
+
+
+def _insert_on_conflict_update(table, conn, keys, data_iter, indexes):
+    data = [dict(zip(keys, row)) for row in data_iter]
+    stmt = insert(table.table).values(data)
+    stmt = stmt.on_conflict_do_update(index_elements=indexes, set_={k: getattr(stmt.excluded, k) for k in keys})
+    result = conn.execute(stmt)
+    return result.rowcount
+
+
+def _insert_on_conflict_nothing(table, conn, keys, data_iter):
+    data = [dict(zip(keys, row)) for row in data_iter]
+    stmt = insert(table.table).values(data).on_conflict_do_nothing(index_elements=keys)
+    result = conn.execute(stmt)
+    return result.rowcount
+
+
+class SqlManager:
+    def __init__(
+        self,
+    ):
+        self.engine = create_engine(CONFIG["sqlconnstr"])
+
+    def add_index(self, table_name: str, columns: Union[str, Sequence[str]], unique: bool = True):
+        if isinstance(columns, str):
+            columns = [columns]
+        index_name = f"uix_{table_name}_{'_'.join(columns)}"
+        columns_str = ", ".join(columns)
+        unique_str = "UNIQUE" if unique else ""
+        self.engine.execute(f"CREATE {unique_str} INDEX {index_name} ON {table_name} ({columns_str})")
+
+    def insert(
+        self,
+        table_name: str,
+        df: pd.DataFrame,
+        upsert: bool = True,
+        other_unique_index_columns: Sequence[str] = (),
+        other_non_unique_index_columns: Sequence[str] = (),
+    ):
+        if_exists: Literal["replace", "append"] = "append"
+        inspector = inspect(self.engine)
+        new_table = not inspector.has_table(table_name)
+        method = (
+            partial(_insert_on_conflict_update, indexes=df.index.names)
+            if upsert and not new_table
+            else None
+        )
+        num_rows = df.to_sql(
+            table_name,
+            self.engine,
+            if_exists=if_exists,
+            index=True,
+            index_label=df.index.names,
+            method=method,
+        )
+        if new_table:
+            self.add_index(table_name, df.index.names)
+            for column in other_unique_index_columns:
+                self.add_index(table_name, column, unique=True)
+            for column in other_non_unique_index_columns:
+                self.add_index(table_name, column, unique=False)
+        return num_rows
+
+    def rename_column(self, table_name: str, old_column_name: str, new_column_name: str):
+        # check if any index is referring to the column
+        sql_code = f"SELECT indexname, indexdef FROM pg_indexes WHERE indexdef LIKE '%%(%%{old_column_name}%%)%%' and tablename = '{table_name}'"
+        index_refering_column = dict(self.engine.execute(sql_code).fetchall())
+
+        # drop indexes referring to the column
+        for index_name in index_refering_column:
+            self.engine.execute(f"DROP INDEX IF EXISTS {index_name}")
+
+        # rename the column
+        self.engine.execute(f"ALTER TABLE {table_name} RENAME COLUMN {old_column_name} TO {new_column_name}")
+
+        # recreate the indexes
+        for index_name, index_def in index_refering_column.items():
+            new_index_name = index_name.replace(old_column_name, new_column_name)
+            new_index_def = re.sub(rf"\(([^)]*?){old_column_name}([^)]*?)\)", rf"(\1{new_column_name}\2)", index_def)
+            new_index_def = new_index_def.replace(index_name, new_index_name)
+            self.engine.execute(new_index_def)
+
+    def read_range_data(
+        self, table_name: str, query_fields, start_time: pd.Timestamp, end_time: pd.Timestamp, filter_fields=None
+    ):
+        sql_code = f"SELECT {', '.join(query_fields)} FROM {table_name} WHERE end_time >= '{start_time}' AND start_time <= '{end_time}'"
+        if filter_fields:
+            sql_code += (
+                f" AND {' AND '.join([f'{field} IN {tuple(filter_values)}' for field, filter_values in filter_fields])}"
+            )
+
+        # cannot use pandas.read_sql here as it discards timezone info
+        res = self.engine.execute(sql_code)
+        return pd.DataFrame(res.fetchall(), columns=res.keys())

{trade_database_manager-0.0.1.dev0 → trade_database_manager-0.0.1.dev1}/trade_database_manager/tools/init_config.py

@@ -12,13 +12,13 @@ if __name__ == "__main__":
         "--host",
         type=str,
         default="localhost",
-        description="host of both kdb and sql, will be overwritten if sqlhost or kdbhost is specified",
+        help="host of both kdb and sql, will be overwritten if sqlhost or kdbhost is specified",
     )
-    parser.add_argument("--sqlhost", type=str, default="", description="host of sql")
-    parser.add_argument("--sqldb", type=str, default="trade_data", description="database name of sql")
-    parser.add_argument("--kdbhost", type=str, default="", description="host of kdb")
-    parser.add_argument("--sqlport", type=str, default="5432", description="port of sql")
-    parser.add_argument("--kdbport", type=str, default="5000", description="port of kdb")
+    parser.add_argument("--sqlhost", type=str, default="", help="host of sql")
+    parser.add_argument("--sqldb", type=str, default="trade_data", help="database name of sql")
+    parser.add_argument("--kdbhost", type=str, default="", help="host of kdb")
+    parser.add_argument("--sqlport", type=str, default="5432", help="port of sql")
+    parser.add_argument("--kdbport", type=str, default="5000", help="port of kdb")
 
     args = parser.parse_args()
 

{trade_database_manager-0.0.1.dev0 → trade_database_manager-0.0.1.dev1/trade_database_manager.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: trade_database_manager
-Version: 0.0.1.dev0
+Version: 0.0.1.dev1
 Summary: A wrapper of kdb and sql for convenient trade data management.
 Author-email: "Y.Q. Cui" <qianyun210603@hotmail.com>
 Classifier: Operating System :: POSIX :: Linux
@@ -11,9 +11,10 @@ Classifier: License :: OSI Approved :: GNU Affero General Public License v3
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pandas
+Requires-Dist: pandas>=2.0.0
 Requires-Dist: numpy
 Requires-Dist: ruamel.yaml
 Requires-Dist: sqlalchemy
@@ -49,3 +50,7 @@ host    all            tradedbadmin     0.0.0.0/0               scram-sha-256
 host    all            tradedbadmin     ::/0                    scram-sha-256
 ```
 
+### KDB+ Initialization
+
+
+```q

{trade_database_manager-0.0.1.dev0 → trade_database_manager-0.0.1.dev1}/trade_database_manager.egg-info/requires.txt

@@ -1,4 +1,4 @@
-pandas
+pandas>=2.0.0
 numpy
 ruamel.yaml
 sqlalchemy

trade_database_manager-0.0.1.dev0/trade_database_manager/core/sql/sqlmanager.py

@@ -1,57 +0,0 @@
-from functools import partial
-from typing import Literal, Sequence, Union
-
-import pandas as pd
-from sqlalchemy import create_engine
-from sqlalchemy.dialects.postgresql import insert
-
-from ...config import CONFIG
-
-
-def _insert_on_conflict_update(table, conn, keys, data_iter, indexes):
-    data = [dict(zip(keys, row)) for row in data_iter]
-    stmt = insert(table.table).values(data)
-    stmt = stmt.on_conflict_do_update(index_elements=indexes, set_={k: getattr(stmt.excluded, k) for k in keys})
-    result = conn.execute(stmt)
-    return result.rowcount
-
-
-def _insert_on_conflict_nothing(table, conn, keys, data_iter):
-    data = [dict(zip(keys, row)) for row in data_iter]
-    stmt = insert(table.table).values(data).on_conflict_do_nothing(index_elements=keys)
-    result = conn.execute(stmt)
-    return result.rowcount
-
-
-class SqlManager:
-    def __init__(
-        self,
-    ):
-        self.engine = create_engine(CONFIG["sqlconnstr"])
-
-    def _add_unique_index(self, table_name: str, columns: Union[str, Sequence[str]]):
-        if isinstance(columns, str):
-            columns = [columns]
-        index_name = f"uix_{table_name}_{'_'.join(columns)}"
-        columns_str = ", ".join(columns)
-        self.engine.execute(f"CREATE UNIQUE INDEX {index_name} ON {table_name} ({columns_str})")
-
-    def insert(self, table_name: str, df: pd.DataFrame, upsert: bool = True):
-        if_exists: Literal["replace", "append"] = "append"
-        method = (
-            partial(_insert_on_conflict_update, indexes=df.index.names)
-            if upsert and self.engine.has_table(table_name)
-            else None
-        )
-        new_table = not self.engine.has_table(table_name)
-        num_rows = df.to_sql(
-            table_name,
-            self.engine,
-            if_exists=if_exists,
-            index=True,
-            index_label=df.index.names,
-            method=method,
-        )
-        if new_table:
-            self._add_unique_index(table_name, df.index.names)
-        return num_rows