meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parse_arguments.py +1 -1
- meerschaum/actions/delete.py +65 -69
- meerschaum/actions/edit.py +22 -2
- meerschaum/actions/install.py +1 -2
- meerschaum/actions/sync.py +2 -3
- meerschaum/api/routes/_pipes.py +7 -8
- meerschaum/config/_default.py +1 -1
- meerschaum/config/_paths.py +2 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_pipes.py +18 -21
- meerschaum/connectors/sql/_create_engine.py +3 -3
- meerschaum/connectors/sql/_instance.py +11 -12
- meerschaum/connectors/sql/_pipes.py +143 -91
- meerschaum/connectors/sql/_sql.py +43 -8
- meerschaum/connectors/valkey/_pipes.py +12 -1
- meerschaum/core/Pipe/__init__.py +23 -13
- meerschaum/core/Pipe/_attributes.py +25 -1
- meerschaum/core/Pipe/_dtypes.py +23 -16
- meerschaum/core/Pipe/_sync.py +59 -31
- meerschaum/core/Pipe/_verify.py +8 -7
- meerschaum/jobs/_Job.py +4 -1
- meerschaum/plugins/_Plugin.py +11 -14
- meerschaum/utils/daemon/Daemon.py +22 -15
- meerschaum/utils/dataframe.py +178 -16
- meerschaum/utils/dtypes/__init__.py +149 -14
- meerschaum/utils/dtypes/sql.py +41 -7
- meerschaum/utils/misc.py +8 -8
- meerschaum/utils/packages/_packages.py +1 -1
- meerschaum/utils/schedule.py +8 -3
- meerschaum/utils/sql.py +180 -100
- meerschaum/utils/venv/_Venv.py +4 -4
- meerschaum/utils/venv/__init__.py +53 -20
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/METADATA +2 -2
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/RECORD +40 -40
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/WHEEL +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/zip-safe +0 -0
    
        meerschaum/utils/sql.py
    CHANGED
    
    | @@ -7,6 +7,7 @@ Flavor-specific SQL tools. | |
| 7 7 | 
             
            """
         | 
| 8 8 |  | 
| 9 9 | 
             
            from __future__ import annotations
         | 
| 10 | 
            +
             | 
| 10 11 | 
             
            from datetime import datetime, timezone, timedelta
         | 
| 11 12 | 
             
            import meerschaum as mrsm
         | 
| 12 13 | 
             
            from meerschaum.utils.typing import Optional, Dict, Any, Union, List, Iterable, Tuple
         | 
| @@ -50,10 +51,12 @@ update_queries = { | |
| 50 51 | 
             
                    {sets_subquery_none}
         | 
| 51 52 | 
             
                    FROM {target_table_name} AS t
         | 
| 52 53 | 
             
                    INNER JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
         | 
| 53 | 
            -
                        ON | 
| 54 | 
            +
                        ON
         | 
| 55 | 
            +
                            {and_subquery_t}
         | 
| 54 56 | 
             
                    WHERE
         | 
| 55 57 | 
             
                        {and_subquery_f}
         | 
| 56 | 
            -
                        AND | 
| 58 | 
            +
                        AND
         | 
| 59 | 
            +
                        {date_bounds_subquery}
         | 
| 57 60 | 
             
                """,
         | 
| 58 61 | 
             
                'timescaledb-upsert': """
         | 
| 59 62 | 
             
                    INSERT INTO {target_table_name} ({patch_cols_str})
         | 
| @@ -82,9 +85,11 @@ update_queries = { | |
| 82 85 | 
             
                'mysql': """
         | 
| 83 86 | 
             
                    UPDATE {target_table_name} AS f
         | 
| 84 87 | 
             
                    JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
         | 
| 85 | 
            -
                    ON | 
| 88 | 
            +
                    ON
         | 
| 89 | 
            +
                        {and_subquery_f}
         | 
| 86 90 | 
             
                    {sets_subquery_f}
         | 
| 87 | 
            -
                    WHERE | 
| 91 | 
            +
                    WHERE
         | 
| 92 | 
            +
                        {date_bounds_subquery}
         | 
| 88 93 | 
             
                """,
         | 
| 89 94 | 
             
                'mysql-upsert': """
         | 
| 90 95 | 
             
                    INSERT {ignore}INTO {target_table_name} ({patch_cols_str})
         | 
| @@ -96,9 +101,11 @@ update_queries = { | |
| 96 101 | 
             
                'mariadb': """
         | 
| 97 102 | 
             
                    UPDATE {target_table_name} AS f
         | 
| 98 103 | 
             
                    JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
         | 
| 99 | 
            -
                    ON | 
| 104 | 
            +
                    ON
         | 
| 105 | 
            +
                        {and_subquery_f}
         | 
| 100 106 | 
             
                    {sets_subquery_f}
         | 
| 101 | 
            -
                    WHERE | 
| 107 | 
            +
                    WHERE
         | 
| 108 | 
            +
                        {date_bounds_subquery}
         | 
| 102 109 | 
             
                """,
         | 
| 103 110 | 
             
                'mariadb-upsert': """
         | 
| 104 111 | 
             
                    INSERT {ignore}INTO {target_table_name} ({patch_cols_str})
         | 
| @@ -108,34 +115,56 @@ update_queries = { | |
| 108 115 | 
             
                        {cols_equal_values}
         | 
| 109 116 | 
             
                """,
         | 
| 110 117 | 
             
                'mssql': """
         | 
| 118 | 
            +
                    {with_temp_date_bounds}
         | 
| 111 119 | 
             
                    MERGE {target_table_name} f
         | 
| 112 | 
            -
                        USING (SELECT  | 
| 113 | 
            -
                        ON | 
| 114 | 
            -
             | 
| 120 | 
            +
                        USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
         | 
| 121 | 
            +
                        ON
         | 
| 122 | 
            +
                            {and_subquery_f}
         | 
| 123 | 
            +
                        AND
         | 
| 124 | 
            +
                            {date_bounds_subquery}
         | 
| 115 125 | 
             
                    WHEN MATCHED THEN
         | 
| 116 126 | 
             
                        UPDATE
         | 
| 117 127 | 
             
                        {sets_subquery_none};
         | 
| 118 128 | 
             
                """,
         | 
| 119 | 
            -
                'mssql-upsert':  | 
| 129 | 
            +
                'mssql-upsert': [
         | 
| 130 | 
            +
                    "{identity_insert_on}",
         | 
| 131 | 
            +
                    """
         | 
| 132 | 
            +
                    {with_temp_date_bounds}
         | 
| 120 133 | 
             
                    MERGE {target_table_name} f
         | 
| 121 | 
            -
                        USING (SELECT  | 
| 122 | 
            -
                        ON | 
| 123 | 
            -
             | 
| 124 | 
            -
             | 
| 134 | 
            +
                        USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
         | 
| 135 | 
            +
                        ON
         | 
| 136 | 
            +
                            {and_subquery_f}
         | 
| 137 | 
            +
                        AND
         | 
| 138 | 
            +
                            {date_bounds_subquery}{when_matched_update_sets_subquery_none}
         | 
| 125 139 | 
             
                    WHEN NOT MATCHED THEN
         | 
| 126 140 | 
             
                        INSERT ({patch_cols_str})
         | 
| 127 141 | 
             
                        VALUES ({patch_cols_prefixed_str});
         | 
| 128 | 
            -
             | 
| 142 | 
            +
                    """,
         | 
| 143 | 
            +
                    "{identity_insert_off}",
         | 
| 144 | 
            +
                ],
         | 
| 129 145 | 
             
                'oracle': """
         | 
| 130 146 | 
             
                    MERGE INTO {target_table_name} f
         | 
| 131 | 
            -
                        USING (SELECT  | 
| 147 | 
            +
                        USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
         | 
| 132 148 | 
             
                        ON (
         | 
| 133 149 | 
             
                            {and_subquery_f}
         | 
| 134 | 
            -
                            AND | 
| 150 | 
            +
                            AND
         | 
| 151 | 
            +
                            {date_bounds_subquery}
         | 
| 135 152 | 
             
                        )
         | 
| 136 | 
            -
             | 
| 137 | 
            -
             | 
| 138 | 
            -
             | 
| 153 | 
            +
                        WHEN MATCHED THEN
         | 
| 154 | 
            +
                            UPDATE
         | 
| 155 | 
            +
                            {sets_subquery_none}
         | 
| 156 | 
            +
                """,
         | 
| 157 | 
            +
                'oracle-upsert': """
         | 
| 158 | 
            +
                    MERGE INTO {target_table_name} f
         | 
| 159 | 
            +
                        USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
         | 
| 160 | 
            +
                        ON (
         | 
| 161 | 
            +
                            {and_subquery_f}
         | 
| 162 | 
            +
                            AND
         | 
| 163 | 
            +
                            {date_bounds_subquery}
         | 
| 164 | 
            +
                        ){when_matched_update_sets_subquery_none}
         | 
| 165 | 
            +
                        WHEN NOT MATCHED THEN
         | 
| 166 | 
            +
                            INSERT ({patch_cols_str})
         | 
| 167 | 
            +
                            VALUES ({patch_cols_prefixed_str})
         | 
| 139 168 | 
             
                """,
         | 
| 140 169 | 
             
                'sqlite-upsert': """
         | 
| 141 170 | 
             
                    INSERT INTO {target_table_name} ({patch_cols_str})
         | 
| @@ -323,7 +352,11 @@ columns_indices_queries = { | |
| 323 352 | 
             
                        CASE
         | 
| 324 353 | 
             
                            WHEN kc.type = 'PK' THEN 'PRIMARY KEY'
         | 
| 325 354 | 
             
                            ELSE 'INDEX'
         | 
| 326 | 
            -
                        END AS [index_type]
         | 
| 355 | 
            +
                        END AS [index_type],
         | 
| 356 | 
            +
                        CASE
         | 
| 357 | 
            +
                            WHEN i.type = 1 THEN CAST(1 AS BIT)
         | 
| 358 | 
            +
                            ELSE CAST(0 AS BIT)
         | 
| 359 | 
            +
                        END AS [clustered]
         | 
| 327 360 | 
             
                    FROM
         | 
| 328 361 | 
             
                        sys.schemas s
         | 
| 329 362 | 
             
                    INNER JOIN sys.tables t
         | 
| @@ -425,20 +458,10 @@ reset_autoincrement_queries: Dict[str, Union[str, List[str]]] = { | |
| 425 458 | 
             
                    SET seq = {val}
         | 
| 426 459 | 
             
                    WHERE name = '{table}'
         | 
| 427 460 | 
             
                """,
         | 
| 428 | 
            -
                'oracle':  | 
| 429 | 
            -
                    "" | 
| 430 | 
            -
                     | 
| 431 | 
            -
             | 
| 432 | 
            -
                        current_val NUMBER;
         | 
| 433 | 
            -
                    BEGIN
         | 
| 434 | 
            -
                        SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
         | 
| 435 | 
            -
             | 
| 436 | 
            -
                        WHILE current_val < max_id LOOP
         | 
| 437 | 
            -
                            SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
         | 
| 438 | 
            -
                        END LOOP;
         | 
| 439 | 
            -
                    END;
         | 
| 440 | 
            -
                    """,
         | 
| 441 | 
            -
                ],
         | 
| 461 | 
            +
                'oracle': (
         | 
| 462 | 
            +
                    "ALTER TABLE {table_name} MODIFY {column_name} "
         | 
| 463 | 
            +
                    "GENERATED BY DEFAULT ON NULL AS IDENTITY (START WITH {val_plus_1})"
         | 
| 464 | 
            +
                ),
         | 
| 442 465 | 
             
            }
         | 
| 443 466 | 
             
            table_wrappers = {
         | 
| 444 467 | 
             
                'default'    : ('"', '"'),
         | 
| @@ -499,7 +522,8 @@ def dateadd_str( | |
| 499 522 | 
             
                flavor: str = 'postgresql',
         | 
| 500 523 | 
             
                datepart: str = 'day',
         | 
| 501 524 | 
             
                number: Union[int, float] = 0,
         | 
| 502 | 
            -
                begin: Union[str, datetime, int] = 'now'
         | 
| 525 | 
            +
                begin: Union[str, datetime, int] = 'now',
         | 
| 526 | 
            +
                db_type: Optional[str] = None,
         | 
| 503 527 | 
             
            ) -> str:
         | 
| 504 528 | 
             
                """
         | 
| 505 529 | 
             
                Generate a `DATEADD` clause depending on database flavor.
         | 
| @@ -538,6 +562,10 @@ def dateadd_str( | |
| 538 562 | 
             
                begin: Union[str, datetime], default `'now'`
         | 
| 539 563 | 
             
                    Base datetime to which to add dateparts.
         | 
| 540 564 |  | 
| 565 | 
            +
                db_type: Optional[str], default None
         | 
| 566 | 
            +
                    If provided, cast the datetime string as the type.
         | 
| 567 | 
            +
                    Otherwise, infer this from the input datetime value.
         | 
| 568 | 
            +
             | 
| 541 569 | 
             
                Returns
         | 
| 542 570 | 
             
                -------
         | 
| 543 571 | 
             
                The appropriate `DATEADD` string for the corresponding database flavor.
         | 
| @@ -549,7 +577,7 @@ def dateadd_str( | |
| 549 577 | 
             
                ...     begin = datetime(2022, 1, 1, 0, 0),
         | 
| 550 578 | 
             
                ...     number = 1,
         | 
| 551 579 | 
             
                ... )
         | 
| 552 | 
            -
                "DATEADD(day, 1, CAST('2022-01-01 00:00:00' AS  | 
| 580 | 
            +
                "DATEADD(day, 1, CAST('2022-01-01 00:00:00' AS DATETIME2))"
         | 
| 553 581 | 
             
                >>> dateadd_str(
         | 
| 554 582 | 
             
                ...     flavor = 'postgresql',
         | 
| 555 583 | 
             
                ...     begin = datetime(2022, 1, 1, 0, 0),
         | 
| @@ -592,7 +620,7 @@ def dateadd_str( | |
| 592 620 | 
             
                    )
         | 
| 593 621 |  | 
| 594 622 | 
             
                dt_is_utc = begin_time.tzinfo is not None if begin_time is not None else '+' in str(begin)
         | 
| 595 | 
            -
                db_type = get_db_type_from_pd_type(
         | 
| 623 | 
            +
                db_type = db_type or get_db_type_from_pd_type(
         | 
| 596 624 | 
             
                    ('datetime64[ns, UTC]' if dt_is_utc else 'datetime64[ns]'),
         | 
| 597 625 | 
             
                    flavor=flavor,
         | 
| 598 626 | 
             
                )
         | 
| @@ -717,7 +745,7 @@ def get_distinct_col_count( | |
| 717 745 | 
             
                result = connector.value(_meta_query, debug=debug)
         | 
| 718 746 | 
             
                try:
         | 
| 719 747 | 
             
                    return int(result)
         | 
| 720 | 
            -
                except Exception | 
| 748 | 
            +
                except Exception:
         | 
| 721 749 | 
             
                    return None
         | 
| 722 750 |  | 
| 723 751 |  | 
| @@ -727,12 +755,15 @@ def sql_item_name(item: str, flavor: str, schema: Optional[str] = None) -> str: | |
| 727 755 |  | 
| 728 756 | 
             
                Parameters
         | 
| 729 757 | 
             
                ----------
         | 
| 730 | 
            -
                item: str | 
| 758 | 
            +
                item: str
         | 
| 731 759 | 
             
                    The database item (table, view, etc.) in need of quotes.
         | 
| 732 760 |  | 
| 733 | 
            -
                flavor: str | 
| 761 | 
            +
                flavor: str
         | 
| 734 762 | 
             
                    The database flavor (`'postgresql'`, `'mssql'`, `'sqllite'`, etc.).
         | 
| 735 763 |  | 
| 764 | 
            +
                schema: Optional[str], default None
         | 
| 765 | 
            +
                    If provided, prefix the table name with the schema.
         | 
| 766 | 
            +
             | 
| 736 767 | 
             
                Returns
         | 
| 737 768 | 
             
                -------
         | 
| 738 769 | 
             
                A `str` which contains the input `item` wrapped in the corresponding escape characters.
         | 
| @@ -764,6 +795,8 @@ def sql_item_name(item: str, flavor: str, schema: Optional[str] = None) -> str: | |
| 764 795 | 
             
                ### NOTE: SQLite does not support schemas.
         | 
| 765 796 | 
             
                if flavor == 'sqlite':
         | 
| 766 797 | 
             
                    schema = None
         | 
| 798 | 
            +
                elif flavor == 'mssql' and str(item).startswith('#'):
         | 
| 799 | 
            +
                    schema = None
         | 
| 767 800 |  | 
| 768 801 | 
             
                schema_prefix = (
         | 
| 769 802 | 
             
                    (wrappers[0] + schema + wrappers[1] + '.')
         | 
| @@ -1066,7 +1099,7 @@ def get_sqlalchemy_table( | |
| 1066 1099 | 
             
                            connector.metadata,
         | 
| 1067 1100 | 
             
                            **table_kwargs
         | 
| 1068 1101 | 
             
                        )
         | 
| 1069 | 
            -
                    except sqlalchemy.exc.NoSuchTableError | 
| 1102 | 
            +
                    except sqlalchemy.exc.NoSuchTableError:
         | 
| 1070 1103 | 
             
                        warn(f"Table '{truncated_table_name}' does not exist in '{connector}'.")
         | 
| 1071 1104 | 
             
                        return None
         | 
| 1072 1105 | 
             
                return tables[truncated_table_name]
         | 
| @@ -1119,6 +1152,7 @@ def get_table_cols_types( | |
| 1119 1152 | 
             
                -------
         | 
| 1120 1153 | 
             
                A dictionary mapping column names to data types.
         | 
| 1121 1154 | 
             
                """
         | 
| 1155 | 
            +
                import textwrap
         | 
| 1122 1156 | 
             
                from meerschaum.connectors import SQLConnector
         | 
| 1123 1157 | 
             
                sqlalchemy = mrsm.attempt_import('sqlalchemy')
         | 
| 1124 1158 | 
             
                flavor = flavor or getattr(connectable, 'flavor', None)
         | 
| @@ -1144,7 +1178,7 @@ def get_table_cols_types( | |
| 1144 1178 | 
             
                )
         | 
| 1145 1179 |  | 
| 1146 1180 | 
             
                cols_types_query = sqlalchemy.text(
         | 
| 1147 | 
            -
                    columns_types_queries.get(
         | 
| 1181 | 
            +
                    textwrap.dedent(columns_types_queries.get(
         | 
| 1148 1182 | 
             
                        flavor,
         | 
| 1149 1183 | 
             
                        columns_types_queries['default']
         | 
| 1150 1184 | 
             
                    ).format(
         | 
| @@ -1155,7 +1189,7 @@ def get_table_cols_types( | |
| 1155 1189 | 
             
                        table_upper=table_upper,
         | 
| 1156 1190 | 
             
                        table_upper_trunc=table_upper_trunc,
         | 
| 1157 1191 | 
             
                        db_prefix=db_prefix,
         | 
| 1158 | 
            -
                    )
         | 
| 1192 | 
            +
                    )).lstrip().rstrip()
         | 
| 1159 1193 | 
             
                )
         | 
| 1160 1194 |  | 
| 1161 1195 | 
             
                cols = ['database', 'schema', 'table', 'column', 'type']
         | 
| @@ -1269,6 +1303,7 @@ def get_table_cols_indices( | |
| 1269 1303 | 
             
                -------
         | 
| 1270 1304 | 
             
                A dictionary mapping column names to a list of indices.
         | 
| 1271 1305 | 
             
                """
         | 
| 1306 | 
            +
                import textwrap
         | 
| 1272 1307 | 
             
                from collections import defaultdict
         | 
| 1273 1308 | 
             
                from meerschaum.connectors import SQLConnector
         | 
| 1274 1309 | 
             
                sqlalchemy = mrsm.attempt_import('sqlalchemy')
         | 
| @@ -1295,7 +1330,7 @@ def get_table_cols_indices( | |
| 1295 1330 | 
             
                )
         | 
| 1296 1331 |  | 
| 1297 1332 | 
             
                cols_indices_query = sqlalchemy.text(
         | 
| 1298 | 
            -
                    columns_indices_queries.get(
         | 
| 1333 | 
            +
                    textwrap.dedent(columns_indices_queries.get(
         | 
| 1299 1334 | 
             
                        flavor,
         | 
| 1300 1335 | 
             
                        columns_indices_queries['default']
         | 
| 1301 1336 | 
             
                    ).format(
         | 
| @@ -1307,10 +1342,12 @@ def get_table_cols_indices( | |
| 1307 1342 | 
             
                        table_upper_trunc=table_upper_trunc,
         | 
| 1308 1343 | 
             
                        db_prefix=db_prefix,
         | 
| 1309 1344 | 
             
                        schema=schema,
         | 
| 1310 | 
            -
                    )
         | 
| 1345 | 
            +
                    )).lstrip().rstrip()
         | 
| 1311 1346 | 
             
                )
         | 
| 1312 1347 |  | 
| 1313 1348 | 
             
                cols = ['database', 'schema', 'table', 'column', 'index', 'index_type']
         | 
| 1349 | 
            +
                if flavor == 'mssql':
         | 
| 1350 | 
            +
                    cols.append('clustered')
         | 
| 1314 1351 | 
             
                result_cols_ix = dict(enumerate(cols))
         | 
| 1315 1352 |  | 
| 1316 1353 | 
             
                debug_kwargs = {'debug': debug} if isinstance(connectable, SQLConnector) else {}
         | 
| @@ -1351,7 +1388,6 @@ def get_table_cols_indices( | |
| 1351 1388 | 
             
                            )
         | 
| 1352 1389 | 
             
                        )
         | 
| 1353 1390 | 
             
                    ]
         | 
| 1354 | 
            -
             | 
| 1355 1391 | 
             
                    ### NOTE: This may return incorrect columns if the schema is not explicitly stated.
         | 
| 1356 1392 | 
             
                    if cols_types_docs and not cols_types_docs_filtered:
         | 
| 1357 1393 | 
             
                        cols_types_docs_filtered = cols_types_docs
         | 
| @@ -1367,12 +1403,13 @@ def get_table_cols_indices( | |
| 1367 1403 | 
             
                                else doc['column']
         | 
| 1368 1404 | 
             
                            )
         | 
| 1369 1405 | 
             
                        )
         | 
| 1370 | 
            -
                         | 
| 1371 | 
            -
                             | 
| 1372 | 
            -
             | 
| 1373 | 
            -
             | 
| 1374 | 
            -
             | 
| 1375 | 
            -
             | 
| 1406 | 
            +
                        index_doc = {
         | 
| 1407 | 
            +
                            'name': doc.get('index', None),
         | 
| 1408 | 
            +
                            'type': doc.get('index_type', None)
         | 
| 1409 | 
            +
                        }
         | 
| 1410 | 
            +
                        if flavor == 'mssql':
         | 
| 1411 | 
            +
                            index_doc['clustered'] = doc.get('clustered', None)
         | 
| 1412 | 
            +
                        cols_indices[col].append(index_doc)
         | 
| 1376 1413 |  | 
| 1377 1414 | 
             
                    return dict(cols_indices)
         | 
| 1378 1415 | 
             
                except Exception as e:
         | 
| @@ -1393,6 +1430,7 @@ def get_update_queries( | |
| 1393 1430 | 
             
                datetime_col: Optional[str] = None,
         | 
| 1394 1431 | 
             
                schema: Optional[str] = None,
         | 
| 1395 1432 | 
             
                patch_schema: Optional[str] = None,
         | 
| 1433 | 
            +
                identity_insert: bool = False,
         | 
| 1396 1434 | 
             
                debug: bool = False,
         | 
| 1397 1435 | 
             
            ) -> List[str]:
         | 
| 1398 1436 | 
             
                """
         | 
| @@ -1430,6 +1468,10 @@ def get_update_queries( | |
| 1430 1468 | 
             
                    If provided, use this schema when quoting the patch table.
         | 
| 1431 1469 | 
             
                    Defaults to `schema`.
         | 
| 1432 1470 |  | 
| 1471 | 
            +
                identity_insert: bool, default False
         | 
| 1472 | 
            +
                    If `True`, include `SET IDENTITY_INSERT` queries before and after the update queries.
         | 
| 1473 | 
            +
                    Only applies for MSSQL upserts.
         | 
| 1474 | 
            +
             | 
| 1433 1475 | 
             
                debug: bool, default False
         | 
| 1434 1476 | 
             
                    Verbosity toggle.
         | 
| 1435 1477 |  | 
| @@ -1437,9 +1479,11 @@ def get_update_queries( | |
| 1437 1479 | 
             
                -------
         | 
| 1438 1480 | 
             
                A list of query strings to perform the update operation.
         | 
| 1439 1481 | 
             
                """
         | 
| 1482 | 
            +
                import textwrap
         | 
| 1440 1483 | 
             
                from meerschaum.connectors import SQLConnector
         | 
| 1441 1484 | 
             
                from meerschaum.utils.debug import dprint
         | 
| 1442 | 
            -
                from meerschaum.utils.dtypes | 
| 1485 | 
            +
                from meerschaum.utils.dtypes import are_dtypes_equal
         | 
| 1486 | 
            +
                from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES, get_pd_type_from_db_type
         | 
| 1443 1487 | 
             
                flavor = flavor or (connectable.flavor if isinstance(connectable, SQLConnector) else None)
         | 
| 1444 1488 | 
             
                if not flavor:
         | 
| 1445 1489 | 
             
                    raise ValueError("Provide a flavor if using a SQLAlchemy session.")
         | 
| @@ -1532,21 +1576,35 @@ def get_update_queries( | |
| 1532 1576 | 
             
                def sets_subquery(l_prefix: str, r_prefix: str):
         | 
| 1533 1577 | 
             
                    if not value_cols:
         | 
| 1534 1578 | 
             
                        return ''
         | 
| 1579 | 
            +
             | 
| 1580 | 
            +
                    cast_func_cols = {
         | 
| 1581 | 
            +
                        c_name: (
         | 
| 1582 | 
            +
                            ('', '', '')
         | 
| 1583 | 
            +
                            if (
         | 
| 1584 | 
            +
                                flavor == 'oracle'
         | 
| 1585 | 
            +
                                and are_dtypes_equal(get_pd_type_from_db_type(c_type), 'bytes')
         | 
| 1586 | 
            +
                            )
         | 
| 1587 | 
            +
                            else (
         | 
| 1588 | 
            +
                                ('CAST(', f" AS {c_type.replace('_', ' ')}", ')')
         | 
| 1589 | 
            +
                                if flavor != 'sqlite'
         | 
| 1590 | 
            +
                                else ('', '', '')
         | 
| 1591 | 
            +
                            )
         | 
| 1592 | 
            +
                        )
         | 
| 1593 | 
            +
                        for c_name, c_type in value_cols
         | 
| 1594 | 
            +
                    }
         | 
| 1535 1595 | 
             
                    return 'SET ' + ',\n'.join([
         | 
| 1536 1596 | 
             
                        (
         | 
| 1537 1597 | 
             
                            l_prefix + sql_item_name(c_name, flavor, None)
         | 
| 1538 1598 | 
             
                            + ' = '
         | 
| 1539 | 
            -
                            +  | 
| 1540 | 
            -
                            + r_prefix
         | 
| 1541 | 
            -
                            +  | 
| 1542 | 
            -
                            +  | 
| 1543 | 
            -
                            + (c_type.replace('_', ' ') if flavor != 'sqlite' else '')
         | 
| 1544 | 
            -
                            + (')' if flavor != 'sqlite' else '')
         | 
| 1599 | 
            +
                            + cast_func_cols[c_name][0]
         | 
| 1600 | 
            +
                            + r_prefix + sql_item_name(c_name, flavor, None)
         | 
| 1601 | 
            +
                            + cast_func_cols[c_name][1]
         | 
| 1602 | 
            +
                            + cast_func_cols[c_name][2]
         | 
| 1545 1603 | 
             
                        ) for c_name, c_type in value_cols
         | 
| 1546 1604 | 
             
                    ])
         | 
| 1547 1605 |  | 
| 1548 1606 | 
             
                def and_subquery(l_prefix: str, r_prefix: str):
         | 
| 1549 | 
            -
                    return '\ | 
| 1607 | 
            +
                    return '\n            AND\n                '.join([
         | 
| 1550 1608 | 
             
                        (
         | 
| 1551 1609 | 
             
                            "COALESCE("
         | 
| 1552 1610 | 
             
                            + l_prefix
         | 
| @@ -1554,7 +1612,7 @@ def get_update_queries( | |
| 1554 1612 | 
             
                            + ", "
         | 
| 1555 1613 | 
             
                            + get_null_replacement(c_type, flavor)
         | 
| 1556 1614 | 
             
                            + ")"
         | 
| 1557 | 
            -
                            + ' | 
| 1615 | 
            +
                            + '\n                =\n                '
         | 
| 1558 1616 | 
             
                            + "COALESCE("
         | 
| 1559 1617 | 
             
                            + r_prefix
         | 
| 1560 1618 | 
             
                            + sql_item_name(c_name, flavor, None)
         | 
| @@ -1564,22 +1622,39 @@ def get_update_queries( | |
| 1564 1622 | 
             
                        ) for c_name, c_type in join_cols_types
         | 
| 1565 1623 | 
             
                    ])
         | 
| 1566 1624 |  | 
| 1625 | 
            +
                skip_query_val = ""
         | 
| 1567 1626 | 
             
                target_table_name = sql_item_name(target, flavor, schema)
         | 
| 1568 1627 | 
             
                patch_table_name = sql_item_name(patch, flavor, patch_schema)
         | 
| 1569 1628 | 
             
                dt_col_name = sql_item_name(datetime_col, flavor, None) if datetime_col else None
         | 
| 1629 | 
            +
                date_bounds_table = patch_table_name if flavor != 'mssql' else '[date_bounds]'
         | 
| 1630 | 
            +
                min_dt_col_name = f"MIN({dt_col_name})" if flavor != 'mssql' else '[Min_dt]'
         | 
| 1631 | 
            +
                max_dt_col_name = f"MAX({dt_col_name})" if flavor != 'mssql' else '[Max_dt]'
         | 
| 1570 1632 | 
             
                date_bounds_subquery = (
         | 
| 1571 | 
            -
                    f"""
         | 
| 1572 | 
            -
             | 
| 1573 | 
            -
             | 
| 1574 | 
            -
                    """
         | 
| 1633 | 
            +
                    f"""f.{dt_col_name} >= (SELECT {min_dt_col_name} FROM {date_bounds_table})
         | 
| 1634 | 
            +
                        AND
         | 
| 1635 | 
            +
                            f.{dt_col_name} <= (SELECT {max_dt_col_name} FROM {date_bounds_table})"""
         | 
| 1575 1636 | 
             
                    if datetime_col
         | 
| 1576 1637 | 
             
                    else "1 = 1"
         | 
| 1577 1638 | 
             
                )
         | 
| 1639 | 
            +
                with_temp_date_bounds = f"""WITH [date_bounds] AS (
         | 
| 1640 | 
            +
                    SELECT MIN({dt_col_name}) AS {min_dt_col_name}, MAX({dt_col_name}) AS {max_dt_col_name}
         | 
| 1641 | 
            +
                    FROM {patch_table_name}
         | 
| 1642 | 
            +
                )""" if datetime_col else ""
         | 
| 1643 | 
            +
                identity_insert_on = (
         | 
| 1644 | 
            +
                    f"SET IDENTITY_INSERT {target_table_name} ON"
         | 
| 1645 | 
            +
                    if identity_insert
         | 
| 1646 | 
            +
                    else skip_query_val
         | 
| 1647 | 
            +
                )
         | 
| 1648 | 
            +
                identity_insert_off = (
         | 
| 1649 | 
            +
                    f"SET IDENTITY_INSERT {target_table_name} OFF"
         | 
| 1650 | 
            +
                    if identity_insert
         | 
| 1651 | 
            +
                    else skip_query_val
         | 
| 1652 | 
            +
                )
         | 
| 1578 1653 |  | 
| 1579 1654 | 
             
                ### NOTE: MSSQL upserts must exclude the update portion if only upserting indices.
         | 
| 1580 1655 | 
             
                when_matched_update_sets_subquery_none = "" if not value_cols else (
         | 
| 1581 | 
            -
                    "WHEN MATCHED THEN"
         | 
| 1582 | 
            -
                    f" | 
| 1656 | 
            +
                    "\n        WHEN MATCHED THEN\n"
         | 
| 1657 | 
            +
                    f"            UPDATE {sets_subquery('', 'p.')}"
         | 
| 1583 1658 | 
             
                )
         | 
| 1584 1659 |  | 
| 1585 1660 | 
             
                cols_equal_values = '\n,'.join(
         | 
| @@ -1595,8 +1670,8 @@ def get_update_queries( | |
| 1595 1670 | 
             
                )
         | 
| 1596 1671 | 
             
                ignore = "IGNORE " if not value_cols else ""
         | 
| 1597 1672 |  | 
| 1598 | 
            -
                 | 
| 1599 | 
            -
                    base_query.format(
         | 
| 1673 | 
            +
                formatted_queries = [
         | 
| 1674 | 
            +
                    textwrap.dedent(base_query.format(
         | 
| 1600 1675 | 
             
                        sets_subquery_none=sets_subquery('', 'p.'),
         | 
| 1601 1676 | 
             
                        sets_subquery_none_excluded=sets_subquery('', 'EXCLUDED.'),
         | 
| 1602 1677 | 
             
                        sets_subquery_f=sets_subquery('f.', 'p.'),
         | 
| @@ -1614,10 +1689,16 @@ def get_update_queries( | |
| 1614 1689 | 
             
                        cols_equal_values=cols_equal_values,
         | 
| 1615 1690 | 
             
                        on_duplicate_key_update=on_duplicate_key_update,
         | 
| 1616 1691 | 
             
                        ignore=ignore,
         | 
| 1617 | 
            -
             | 
| 1692 | 
            +
                        with_temp_date_bounds=with_temp_date_bounds,
         | 
| 1693 | 
            +
                        identity_insert_on=identity_insert_on,
         | 
| 1694 | 
            +
                        identity_insert_off=identity_insert_off,
         | 
| 1695 | 
            +
                    )).lstrip().rstrip()
         | 
| 1618 1696 | 
             
                    for base_query in base_queries
         | 
| 1619 1697 | 
             
                ]
         | 
| 1620 1698 |  | 
| 1699 | 
            +
                ### NOTE: Allow for skipping some queries.
         | 
| 1700 | 
            +
                return [query for query in formatted_queries if query]
         | 
| 1701 | 
            +
             | 
| 1621 1702 |  | 
| 1622 1703 | 
             
            def get_null_replacement(typ: str, flavor: str) -> str:
         | 
| 1623 1704 | 
             
                """
         | 
| @@ -1655,11 +1736,14 @@ def get_null_replacement(typ: str, flavor: str) -> str: | |
| 1655 1736 | 
             
                    )
         | 
| 1656 1737 | 
             
                    return f'CAST({val_to_cast} AS {bool_typ})'
         | 
| 1657 1738 | 
             
                if 'time' in typ.lower() or 'date' in typ.lower():
         | 
| 1658 | 
            -
                     | 
| 1739 | 
            +
                    db_type = typ if typ.isupper() else None
         | 
| 1740 | 
            +
                    return dateadd_str(flavor=flavor, begin='1900-01-01', db_type=db_type)
         | 
| 1659 1741 | 
             
                if 'float' in typ.lower() or 'double' in typ.lower() or typ.lower() in ('decimal',):
         | 
| 1660 1742 | 
             
                    return '-987654321.0'
         | 
| 1661 1743 | 
             
                if flavor == 'oracle' and typ.lower().split('(', maxsplit=1)[0] == 'char':
         | 
| 1662 1744 | 
             
                    return "'-987654321'"
         | 
| 1745 | 
            +
                if flavor == 'oracle' and typ.lower() in ('blob', 'bytes'):
         | 
| 1746 | 
            +
                    return '00'
         | 
| 1663 1747 | 
             
                if typ.lower() in ('uniqueidentifier', 'guid', 'uuid'):
         | 
| 1664 1748 | 
             
                    magic_val = 'DEADBEEF-ABBA-BABE-CAFE-DECAFC0FFEE5'
         | 
| 1665 1749 | 
             
                    if flavor == 'mssql':
         | 
| @@ -1867,7 +1951,17 @@ def _get_create_table_query_from_dtypes( | |
| 1867 1951 |  | 
| 1868 1952 | 
             
                table_name = sql_item_name(new_table, schema=schema, flavor=flavor)
         | 
| 1869 1953 | 
             
                primary_key_name = sql_item_name(primary_key, flavor) if primary_key else None
         | 
| 1954 | 
            +
                primary_key_constraint_name = (
         | 
| 1955 | 
            +
                    sql_item_name(f'PK_{new_table}', flavor, None)
         | 
| 1956 | 
            +
                    if primary_key
         | 
| 1957 | 
            +
                    else None
         | 
| 1958 | 
            +
                )
         | 
| 1870 1959 | 
             
                datetime_column_name = sql_item_name(datetime_column, flavor) if datetime_column else None
         | 
| 1960 | 
            +
                primary_key_clustered = (
         | 
| 1961 | 
            +
                    "CLUSTERED"
         | 
| 1962 | 
            +
                    if not datetime_column or datetime_column == primary_key
         | 
| 1963 | 
            +
                    else "NONCLUSTERED"
         | 
| 1964 | 
            +
                )
         | 
| 1871 1965 | 
             
                query = f"CREATE TABLE {table_name} ("
         | 
| 1872 1966 | 
             
                if primary_key:
         | 
| 1873 1967 | 
             
                    col_db_type = cols_types[0][1]
         | 
| @@ -1887,6 +1981,8 @@ def _get_create_table_query_from_dtypes( | |
| 1887 1981 | 
             
                        query += f"\n    {col_name} {col_db_type} {auto_increment_str} PRIMARY KEY,"
         | 
| 1888 1982 | 
             
                    elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
         | 
| 1889 1983 | 
             
                        query += f"\n    {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
         | 
| 1984 | 
            +
                    elif flavor == 'mssql':
         | 
| 1985 | 
            +
                        query += f"\n    {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
         | 
| 1890 1986 | 
             
                    else:
         | 
| 1891 1987 | 
             
                        query += f"\n    {col_name} {col_db_type} PRIMARY KEY{auto_increment_str} NOT NULL,"
         | 
| 1892 1988 |  | 
| @@ -1902,6 +1998,10 @@ def _get_create_table_query_from_dtypes( | |
| 1902 1998 | 
             
                    and datetime_column != primary_key
         | 
| 1903 1999 | 
             
                ):
         | 
| 1904 2000 | 
             
                    query += f"\n    PRIMARY KEY({datetime_column_name}, {primary_key_name}),"
         | 
| 2001 | 
            +
             | 
| 2002 | 
            +
                if flavor == 'mssql' and primary_key:
         | 
| 2003 | 
            +
                    query += f"\n    CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name}),"
         | 
| 2004 | 
            +
             | 
| 1905 2005 | 
             
                query = query[:-1]
         | 
| 1906 2006 | 
             
                query += "\n)"
         | 
| 1907 2007 |  | 
| @@ -1922,12 +2022,11 @@ def _get_create_table_query_from_cte( | |
| 1922 2022 | 
             
                Create a new table from a CTE query.
         | 
| 1923 2023 | 
             
                """
         | 
| 1924 2024 | 
             
                import textwrap
         | 
| 1925 | 
            -
                from meerschaum.utils.dtypes.sql import AUTO_INCREMENT_COLUMN_FLAVORS
         | 
| 1926 2025 | 
             
                create_cte = 'create_query'
         | 
| 1927 2026 | 
             
                create_cte_name = sql_item_name(create_cte, flavor, None)
         | 
| 1928 2027 | 
             
                new_table_name = sql_item_name(new_table, flavor, schema)
         | 
| 1929 2028 | 
             
                primary_key_constraint_name = (
         | 
| 1930 | 
            -
                    sql_item_name(f' | 
| 2029 | 
            +
                    sql_item_name(f'PK_{new_table}', flavor, None)
         | 
| 1931 2030 | 
             
                    if primary_key
         | 
| 1932 2031 | 
             
                    else None
         | 
| 1933 2032 | 
             
                )
         | 
| @@ -1936,6 +2035,7 @@ def _get_create_table_query_from_cte( | |
| 1936 2035 | 
             
                    if primary_key
         | 
| 1937 2036 | 
             
                    else None
         | 
| 1938 2037 | 
             
                )
         | 
| 2038 | 
            +
                primary_key_clustered = "CLUSTERED" if not datetime_column else "NONCLUSTERED"
         | 
| 1939 2039 | 
             
                datetime_column_name = (
         | 
| 1940 2040 | 
             
                    sql_item_name(datetime_column, flavor)
         | 
| 1941 2041 | 
             
                    if datetime_column
         | 
| @@ -1943,7 +2043,7 @@ def _get_create_table_query_from_cte( | |
| 1943 2043 | 
             
                )
         | 
| 1944 2044 | 
             
                if flavor in ('mssql',):
         | 
| 1945 2045 | 
             
                    query = query.lstrip()
         | 
| 1946 | 
            -
                    if  | 
| 2046 | 
            +
                    if query.lower().startswith('with '):
         | 
| 1947 2047 | 
             
                        final_select_ix = query.lower().rfind('select')
         | 
| 1948 2048 | 
             
                        create_table_query = (
         | 
| 1949 2049 | 
             
                            query[:final_select_ix].rstrip() + ',\n'
         | 
| @@ -1961,7 +2061,7 @@ def _get_create_table_query_from_cte( | |
| 1961 2061 |  | 
| 1962 2062 | 
             
                    alter_type_query = f"""
         | 
| 1963 2063 | 
             
                        ALTER TABLE {new_table_name}
         | 
| 1964 | 
            -
                        ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY ({primary_key_name})
         | 
| 2064 | 
            +
                        ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name})
         | 
| 1965 2065 | 
             
                    """
         | 
| 1966 2066 | 
             
                elif flavor in (None,):
         | 
| 1967 2067 | 
             
                    create_table_query = f"""
         | 
| @@ -2009,11 +2109,11 @@ def _get_create_table_query_from_cte( | |
| 2009 2109 | 
             
                        ADD PRIMARY KEY ({primary_key_name})
         | 
| 2010 2110 | 
             
                    """
         | 
| 2011 2111 |  | 
| 2012 | 
            -
                create_table_query = textwrap.dedent(create_table_query)
         | 
| 2112 | 
            +
                create_table_query = textwrap.dedent(create_table_query).lstrip().rstrip()
         | 
| 2013 2113 | 
             
                if not primary_key:
         | 
| 2014 2114 | 
             
                    return [create_table_query]
         | 
| 2015 2115 |  | 
| 2016 | 
            -
                alter_type_query = textwrap.dedent(alter_type_query)
         | 
| 2116 | 
            +
                alter_type_query = textwrap.dedent(alter_type_query).lstrip().rstrip()
         | 
| 2017 2117 |  | 
| 2018 2118 | 
             
                return [
         | 
| 2019 2119 | 
             
                    create_table_query,
         | 
| @@ -2225,29 +2325,8 @@ def get_reset_autoincrement_queries( | |
| 2225 2325 | 
             
                schema = schema or connector.schema
         | 
| 2226 2326 | 
             
                max_id_name = sql_item_name('max_id', connector.flavor)
         | 
| 2227 2327 | 
             
                table_name = sql_item_name(table, connector.flavor, schema)
         | 
| 2228 | 
            -
                table_trunc = truncate_item_name(table, connector.flavor)
         | 
| 2229 2328 | 
             
                table_seq_name = sql_item_name(table + '_' + column + '_seq', connector.flavor, schema)
         | 
| 2230 2329 | 
             
                column_name = sql_item_name(column, connector.flavor)
         | 
| 2231 | 
            -
                if connector.flavor == 'oracle':
         | 
| 2232 | 
            -
                    potential_table_names = set([
         | 
| 2233 | 
            -
                        f"'{table_trunc.upper()}'",
         | 
| 2234 | 
            -
                        f"'{table_trunc}'",
         | 
| 2235 | 
            -
                        f"'{table_name}'",
         | 
| 2236 | 
            -
                        f"'{table_name.upper()}'",
         | 
| 2237 | 
            -
                    ])
         | 
| 2238 | 
            -
                    df = connector.read(
         | 
| 2239 | 
            -
                        """
         | 
| 2240 | 
            -
                        SELECT SEQUENCE_NAME
         | 
| 2241 | 
            -
                        FROM ALL_TAB_IDENTITY_COLS
         | 
| 2242 | 
            -
                        WHERE TABLE_NAME IN ("""
         | 
| 2243 | 
            -
                        + ", ".join([name for name in potential_table_names])
         | 
| 2244 | 
            -
                        + """)
         | 
| 2245 | 
            -
                        """,
         | 
| 2246 | 
            -
                        debug=debug
         | 
| 2247 | 
            -
                    )
         | 
| 2248 | 
            -
                    if len(df) > 0:
         | 
| 2249 | 
            -
                        table_seq_name = df['sequence_name'][0]
         | 
| 2250 | 
            -
             | 
| 2251 2330 | 
             
                max_id = connector.value(
         | 
| 2252 2331 | 
             
                    f"""
         | 
| 2253 2332 | 
             
                    SELECT COALESCE(MAX({column_name}), 0) AS {max_id_name}
         | 
| @@ -2272,7 +2351,8 @@ def get_reset_autoincrement_queries( | |
| 2272 2351 | 
             
                        table=table,
         | 
| 2273 2352 | 
             
                        table_name=table_name,
         | 
| 2274 2353 | 
             
                        table_seq_name=table_seq_name,
         | 
| 2275 | 
            -
                        val= | 
| 2354 | 
            +
                        val=max_id,
         | 
| 2355 | 
            +
                        val_plus_1=(max_id + 1),
         | 
| 2276 2356 | 
             
                    )
         | 
| 2277 2357 | 
             
                    for query in reset_queries
         | 
| 2278 2358 | 
             
                ]
         | 
    
        meerschaum/utils/venv/_Venv.py
    CHANGED
    
    | @@ -34,10 +34,10 @@ class Venv: | |
| 34 34 | 
             
                """
         | 
| 35 35 |  | 
| 36 36 | 
             
                def __init__(
         | 
| 37 | 
            -
             | 
| 38 | 
            -
             | 
| 39 | 
            -
             | 
| 40 | 
            -
             | 
| 37 | 
            +
                    self,
         | 
| 38 | 
            +
                    venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
         | 
| 39 | 
            +
                    debug: bool = False,
         | 
| 40 | 
            +
                ) -> None:
         | 
| 41 41 | 
             
                    from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
         | 
| 42 42 | 
             
                    ### For some weird threading issue,
         | 
| 43 43 | 
             
                    ### we can't use `isinstance` here.
         |