mtsql 1.12.31__py3-none-any.whl → 1.12.33__py3-none-any.whl

mt/sql/base.py

@@ -25,12 +25,17 @@ __all__ = [
     "list_views",
     "table_exists",
     "create_temp_id_table",
+    "create_temp_id_tuple_table",
+    "create_temp_str_id_table",
     "temp_table_name",
     "temp_table_find_new_id",
     "temp_table_drop",
     "to_temp_table",
     "find_common_ids",
+    "find_common_id_tuples",
+    "find_common_str_ids",
     "remove_records_by_id",
+    "remove_records_by_str_id",
 ]
 
 
@@ -420,6 +425,108 @@ def create_temp_id_table(
     return table_name
 
 
+def create_temp_id_tuple_table(
+    la_ids: list,
+    n_cols: int,
+    conn: sa.engine.Connection,
+    int_type: str = "int",
+    chunksize: int = 1000000,
+    logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
+) -> str:
+    """Creates a temporary table to containing a list of multi-column ids.
+
+    Parameters
+    ----------
+    la_ids : list
+        list of fixed-sized arrays of ids to be inserted into the table. Each array must have
+        length equal to `n_cols`.
+    n_cols : int
+        number of columns in the id arrays
+    conn : sqlalchemy.engine.Connection
+        a connection that has been opened
+    int_type : str
+        an SQL string representing the int type
+    chunksize : int
+        maximum number of ids to be inserted in each INSERT statement
+    logger : mt.logg.IndentedLoggerAdapter, optional
+        logger for debugging
+
+    Returns
+    -------
+    table_name : str
+        name of the temporary table. The table will be deleted at the end of the connection
+    """
+
+    if n_cols <= 1:
+        raise ValueError("n_cols must be greater than 1")
+
+    table_name = f"tab_{uuid.uuid4().hex}"
+
+    query_str = f"CREATE TEMP TABLE {table_name}("
+    query_str += ",".join((f"id{i+1} {int_type}" for i in range(n_cols)))
+    query_str += ");"
+    exec_sql(sa.text(query_str), conn, logger=logger)
+
+    while True:
+        la_ids2 = la_ids[:chunksize]
+        if len(la_ids2) == 0:
+            break
+        values = ",".join(
+            "(" + ",".join((str(id[i]) for i in range(n_cols))) + ")" for id in la_ids2
+        )
+        query_str = f"INSERT INTO {table_name}("
+        query_str += ",".join((f"id{i+1}" for i in range(n_cols)))
+        query_str += f") VALUES {values};"
+        exec_sql(sa.text(query_str), conn, logger=logger)
+
+        la_ids = la_ids[chunksize:]
+
+    return table_name
+
+
+def create_temp_str_id_table(
+    l_ids: list,
+    conn: sa.engine.Connection,
+    chunksize: int = 1000000,
+    logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
+) -> str:
+    """Creates a temporary table to containing a list of string ids.
+
+    Parameters
+    ----------
+    l_ids : list
+        list of string ids to be inserted into the table
+    conn : sqlalchemy.engine.Connection
+        a connection that has been opened
+    chunksize : int
+        maximum number of ids to be inserted in each INSERT statement
+    logger : mt.logg.IndentedLoggerAdapter, optional
+        logger for debugging
+
+    Returns
+    -------
+    table_name : str
+        name of the temporary table. The table will be deleted at the end of the connection
+    """
+
+    table_name = f"tab_{uuid.uuid4().hex}"
+
+    query_str = f"CREATE TEMP TABLE {table_name}(id character varying);"
+    exec_sql(sa.text(query_str), conn, logger=logger)
+
+    while True:
+        l_ids2 = l_ids[:chunksize]
+        if len(l_ids2) == 0:
+            break
+
+        values = ",".join((f"('{id}')" for id in l_ids2))
+        query_str = f"INSERT INTO {table_name}(id) VALUES {values};"
+        exec_sql(sa.text(query_str), conn, logger=logger)
+        l_ids = l_ids[chunksize:]
+
+    return table_name
+
+
 def temp_table_name(id: int) -> str:
     """Converts a temp table id into a temp table name."""
     return f"mttmp_{id}"
@@ -550,6 +657,109 @@ def find_common_ids(
     return l_commonIds
 
 
+def find_common_id_tuples(
+    la_ids: tp.List[tp.List[int]],
+    id_cols: tp.List[str],
+    frame_name: str,
+    engine: sa.engine.Engine,
+    schema: tp.Optional[str] = None,
+    int_type: str = "int",
+    logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
+) -> tp.List[int]:
+    """Finds common ids between a list of ids and the ids in a given frame.
+
+    Parameters
+    ----------
+    la_ids : list of list of int
+        list of id tuples to be checked
+    id_cols : list of str
+        name of the id columns in the frame
+    frame_name : str
+        name of the frame to be checked against
+    engine : sqlalchemy.engine.Engine
+        connection engine to the server
+    schema : str, optional
+        schema of the frame. If None, the default schema is used.
+    id_cols : list of str
+        name of the id column in the frame
+    int_type : str
+        an SQL string representing the int type of the id column
+    logger : mt.logg.IndentedLoggerAdapter, optional
+        logger for debugging
+
+    Returns
+    -------
+    list of int
+        list of common ids
+    """
+
+    with conn_ctx(engine) as conn:
+        temp_table = create_temp_id_table(l_ids, conn, int_type=int_type, logger=logger)
+
+        full_frame_name = frame_sql(frame_name, schema=schema)
+
+        sql = f"""
+        SELECT t.id FROM {temp_table} AS t
+        INNER JOIN {full_frame_name} AS f
+        ON t.id = f.{id_col};
+        """
+
+        df_common = read_sql(sql, conn, index_col=None, logger=logger)
+
+        l_commonIds = df_common["id"].tolist()
+
+    return l_commonIds
+
+
+def find_common_str_ids(
+    l_ids: tp.List[str],
+    frame_name: str,
+    engine: sa.engine.Engine,
+    schema: tp.Optional[str] = None,
+    id_col: str = "uuid",
+    logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
+) -> tp.List[int]:
+    """Finds common string ids between a list of string ids and the string ids in a given frame.
+
+    Parameters
+    ----------
+    l_ids : list of strings
+        list of string ids to be checked
+    frame_name : str
+        name of the frame to be checked against
+    engine : sqlalchemy.engine.Engine
+        connection engine to the server
+    schema : str, optional
+        schema of the frame. If None, the default schema is used.
+    id_col : str
+        name of the string id column in the frame
+    logger : mt.logg.IndentedLoggerAdapter, optional
+        logger for debugging
+
+    Returns
+    -------
+    list of strings
+        list of common string ids
+    """
+
+    with conn_ctx(engine) as conn:
+        temp_table = create_temp_str_id_table(l_ids, conn, logger=logger)
+
+        full_frame_name = frame_sql(frame_name, schema=schema)
+
+        sql = f"""
+        SELECT t.id FROM {temp_table} AS t
+        INNER JOIN {full_frame_name} AS f
+        ON t.id = f.{id_col};
+        """
+
+        df_common = read_sql(sql, conn, index_col=None, logger=logger)
+
+        l_commonIds = df_common["id"].tolist()
+
+    return l_commonIds
+
+
 def remove_records_by_id(
     l_ids: tp.List[int],
     frame_name: str,
@@ -591,3 +801,43 @@ def remove_records_by_id(
         """
 
         exec_sql(sql, conn, logger=logger)
+
+
+def remove_records_by_str_id(
+    l_ids: tp.List[str],
+    frame_name: str,
+    engine: sa.engine.Engine,
+    schema: tp.Optional[str] = None,
+    id_col: str = "id",
+    logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
+):
+    """Removes records from a frame by a list of string ids.
+
+    Parameters
+    ----------
+    l_ids : list of strings
+        list of string ids to be removed
+    frame_name : str
+        name of the frame to be modified
+    engine : sqlalchemy.engine.Engine
+        connection engine to the server
+    schema : str, optional
+        schema of the frame. If None, the default schema is used.
+    id_col : str
+        name of the id column in the frame
+    logger : mt.logg.IndentedLoggerAdapter, optional
+        logger for debugging
+    """
+
+    with conn_ctx(engine) as conn:
+        temp_table = create_temp_str_id_table(l_ids, conn, logger=logger)
+
+        full_frame_name = frame_sql(frame_name, schema=schema)
+
+        sql = f"""
+        DELETE FROM {full_frame_name}
+        USING {temp_table} AS t
+        WHERE {full_frame_name}.{id_col} = t.id;
+        """
+
+        exec_sql(sql, conn, logger=logger)

mt/sql/version.py

@@ -1,5 +1,5 @@
 MAJOR_VERSION = 1
 MINOR_VERSION = 12
-PATCH_VERSION = 31
+PATCH_VERSION = 33
 version = '{}.{}.{}'.format(MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION)
 __all__  = ['MAJOR_VERSION', 'MINOR_VERSION', 'PATCH_VERSION', 'version']

{mtsql-1.12.31.dist-info → mtsql-1.12.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mtsql
-Version: 1.12.31
+Version: 1.12.33
 Summary: Extra Python modules to deal with the interaction between pandas dataframes and remote SQL servers, for Minh-Tri Pham
 Home-page: https://github.com/inteplus/mtsql
 Author: Minh-Tri Pham
@@ -12,8 +12,8 @@ Requires-Dist: tzlocal
 Requires-Dist: tqdm
 Requires-Dist: psycopg[binary]
 Requires-Dist: redshift_connector>=2.1.5
-Requires-Dist: mtbase>=4.3.2
-Requires-Dist: mtpandas>=1.17.16
+Requires-Dist: mtbase>=4.33.15
+Requires-Dist: mtpandas>=1.17.20
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{mtsql-1.12.31.dist-info → mtsql-1.12.33.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mt/sql/__init__.py,sha256=b7zO50apZxt9Hg2eOkJhRLrXgACR8eS5b-Rphdn5qNQ,44
-mt/sql/base.py,sha256=CLFKd-rJtIUGLm37WtxeFLzhX48LgGpEoz6noR4qfXo,16510
+mt/sql/base.py,sha256=HJzMlKO-SX3aLTMnOrvhrInfNi9O7WMD3tuV2yZIdJg,23863
 mt/sql/mysql.py,sha256=n2ENDctdUqZuSaDAcrqZYtPtawq3Wx4dOPCRsCB5Q4w,4894
 mt/sql/psql.py,sha256=U8XEyg4rQYr5gm8KohRWrpCNJKl5WC1yxJMBkkm1k_A,68125
 mt/sql/sqlite.py,sha256=T2ak_hhNi_zRfpg_gp8JhNHn7D2kl4i-Ey6-9ANMtz0,8678
-mt/sql/version.py,sha256=e7OodvUPDd0qLszM8zWrGxkmyhHFVodAR8B3CnSzs7g,208
+mt/sql/version.py,sha256=3roEiAIfAeKb5dA-3ZmquKI_ZMs8TFD5x-PCcvQBLe4,208
 mt/sql/redshift/__init__.py,sha256=S-eRxJWcrvncF7LZXuulCdPV-UERu9eiw6uyDb5aGsM,443
 mt/sql/redshift/commands.py,sha256=xhGUBf3bL66EYdZI5HCUtOx-XqPCnXT_P-LnhPgtzrY,40193
 mt/sql/redshift/ddl.py,sha256=eUcZj9oIajiE1wKKBAP-V64gYJ7cVnSAt8dLFfluOJA,9777
 mt/sql/redshift/dialect.py,sha256=-0JjJubZZHRw0abhl6H6rKWaUE9pKtGwAuX-62T0e_c,53399
 mt/sql/redshift/main.py,sha256=H8_5sjtJ7dzWoCMXzPNjYhrMQ18eLUQ9xg-aYl5QeTc,17104
 mt/sql/redshift/redshift-ca-bundle.crt,sha256=532qYkOpQOstFE0mdXE1GVtL3v00XDKgZNTr6gK5-KE,8621
-mtsql-1.12.31.dist-info/licenses/LICENSE,sha256=PojkRlQzTT5Eg6Nj03XoIVEefN3u8iiIFf1p4rqe_t4,1070
-mtsql-1.12.31.dist-info/METADATA,sha256=bJET1fnqtrkAZ1Qz8UzM7c2E0PL1X65I5AN1h6wlY1Q,734
-mtsql-1.12.31.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mtsql-1.12.31.dist-info/top_level.txt,sha256=WcqGFu9cV7iMZg09iam8eNxUvGpLSKKF2Iubf6SJVOo,3
-mtsql-1.12.31.dist-info/RECORD,,
+mtsql-1.12.33.dist-info/licenses/LICENSE,sha256=PojkRlQzTT5Eg6Nj03XoIVEefN3u8iiIFf1p4rqe_t4,1070
+mtsql-1.12.33.dist-info/METADATA,sha256=MEQHkGnmBIr4lwNZrMKlCCbUNUOmprFJXE3a0xt5Slg,736
+mtsql-1.12.33.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+mtsql-1.12.33.dist-info/top_level.txt,sha256=WcqGFu9cV7iMZg09iam8eNxUvGpLSKKF2Iubf6SJVOo,3
+mtsql-1.12.33.dist-info/RECORD,,

{mtsql-1.12.31.dist-info → mtsql-1.12.33.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any