meerschaum 2.3.6__py3-none-any.whl → 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parse_arguments.py +2 -5
- meerschaum/_internal/docs/index.py +3 -2
- meerschaum/_internal/entry.py +13 -7
- meerschaum/_internal/shell/Shell.py +38 -44
- meerschaum/_internal/term/TermPageHandler.py +2 -3
- meerschaum/_internal/term/__init__.py +13 -11
- meerschaum/actions/api.py +10 -7
- meerschaum/actions/bootstrap.py +38 -11
- meerschaum/actions/copy.py +3 -3
- meerschaum/actions/delete.py +4 -1
- meerschaum/actions/register.py +1 -3
- meerschaum/actions/stack.py +24 -19
- meerschaum/actions/start.py +38 -40
- meerschaum/actions/sync.py +53 -52
- meerschaum/api/__init__.py +48 -14
- meerschaum/api/_events.py +15 -10
- meerschaum/api/_oauth2.py +2 -2
- meerschaum/api/_websockets.py +5 -4
- meerschaum/api/dash/__init__.py +7 -16
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/dashboard.py +52 -58
- meerschaum/api/dash/callbacks/jobs.py +15 -16
- meerschaum/api/dash/callbacks/login.py +16 -10
- meerschaum/api/dash/callbacks/pipes.py +41 -0
- meerschaum/api/dash/callbacks/plugins.py +1 -1
- meerschaum/api/dash/callbacks/register.py +15 -11
- meerschaum/api/dash/components.py +54 -59
- meerschaum/api/dash/jobs.py +5 -9
- meerschaum/api/dash/pages/__init__.py +1 -0
- meerschaum/api/dash/pages/pipes.py +19 -0
- meerschaum/api/dash/pipes.py +86 -58
- meerschaum/api/dash/plugins.py +6 -4
- meerschaum/api/dash/sessions.py +176 -0
- meerschaum/api/dash/users.py +3 -41
- meerschaum/api/dash/webterm.py +12 -17
- meerschaum/api/resources/static/js/terminado.js +1 -1
- meerschaum/api/routes/_actions.py +4 -20
- meerschaum/api/routes/_jobs.py +8 -7
- meerschaum/api/routes/_login.py +4 -4
- meerschaum/api/routes/_pipes.py +3 -3
- meerschaum/api/routes/_webterm.py +5 -6
- meerschaum/config/_default.py +15 -2
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +64 -19
- meerschaum/config/static/__init__.py +4 -0
- meerschaum/connectors/{Connector.py → _Connector.py} +19 -13
- meerschaum/connectors/__init__.py +24 -14
- meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
- meerschaum/connectors/api/__init__.py +2 -1
- meerschaum/connectors/parse.py +18 -16
- meerschaum/connectors/poll.py +30 -24
- meerschaum/connectors/sql/__init__.py +3 -1
- meerschaum/connectors/sql/_pipes.py +172 -197
- meerschaum/connectors/sql/_plugins.py +45 -43
- meerschaum/connectors/sql/_users.py +46 -38
- meerschaum/connectors/valkey/_ValkeyConnector.py +535 -0
- meerschaum/connectors/valkey/__init__.py +10 -0
- meerschaum/connectors/valkey/_fetch.py +75 -0
- meerschaum/connectors/valkey/_pipes.py +844 -0
- meerschaum/connectors/valkey/_plugins.py +265 -0
- meerschaum/connectors/valkey/_users.py +305 -0
- meerschaum/core/Pipe/__init__.py +3 -0
- meerschaum/core/Pipe/_attributes.py +1 -2
- meerschaum/core/Pipe/_clear.py +16 -13
- meerschaum/core/Pipe/_copy.py +106 -0
- meerschaum/core/Pipe/_data.py +165 -101
- meerschaum/core/Pipe/_drop.py +4 -4
- meerschaum/core/Pipe/_dtypes.py +14 -14
- meerschaum/core/Pipe/_edit.py +15 -14
- meerschaum/core/Pipe/_sync.py +134 -53
- meerschaum/core/Pipe/_verify.py +11 -11
- meerschaum/core/User/_User.py +14 -12
- meerschaum/jobs/_Job.py +1 -6
- meerschaum/jobs/__init__.py +7 -2
- meerschaum/plugins/_Plugin.py +17 -13
- meerschaum/utils/_get_pipes.py +14 -20
- meerschaum/utils/dataframe.py +291 -101
- meerschaum/utils/dtypes/__init__.py +31 -6
- meerschaum/utils/dtypes/sql.py +4 -4
- meerschaum/utils/formatting/_shell.py +5 -6
- meerschaum/utils/misc.py +3 -3
- meerschaum/utils/packages/__init__.py +14 -9
- meerschaum/utils/packages/_packages.py +2 -0
- meerschaum/utils/schedule.py +1 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/METADATA +7 -1
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/RECORD +93 -84
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/WHEEL +1 -1
- meerschaum/api/dash/actions.py +0 -255
- /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/zip-safe +0 -0
@@ -15,10 +15,10 @@ from meerschaum.utils.warnings import warn, error, info
|
|
15
15
|
from meerschaum.utils.debug import dprint
|
16
16
|
|
17
17
|
def register_pipe(
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
18
|
+
self,
|
19
|
+
pipe: mrsm.Pipe,
|
20
|
+
debug: bool = False,
|
21
|
+
) -> SuccessTuple:
|
22
22
|
"""
|
23
23
|
Register a new pipe.
|
24
24
|
A pipe's attributes must be set before registering.
|
@@ -140,14 +140,14 @@ def edit_pipe(
|
|
140
140
|
|
141
141
|
|
142
142
|
def fetch_pipes_keys(
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
143
|
+
self,
|
144
|
+
connector_keys: Optional[List[str]] = None,
|
145
|
+
metric_keys: Optional[List[str]] = None,
|
146
|
+
location_keys: Optional[List[str]] = None,
|
147
|
+
tags: Optional[List[str]] = None,
|
148
|
+
params: Optional[Dict[str, Any]] = None,
|
149
|
+
debug: bool = False
|
150
|
+
) -> Optional[List[Tuple[str, str, Optional[str]]]]:
|
151
151
|
"""
|
152
152
|
Return a list of tuples corresponding to the parameters provided.
|
153
153
|
|
@@ -634,20 +634,20 @@ def delete_pipe(
|
|
634
634
|
|
635
635
|
|
636
636
|
def get_pipe_data(
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
637
|
+
self,
|
638
|
+
pipe: mrsm.Pipe,
|
639
|
+
select_columns: Optional[List[str]] = None,
|
640
|
+
omit_columns: Optional[List[str]] = None,
|
641
|
+
begin: Union[datetime, str, None] = None,
|
642
|
+
end: Union[datetime, str, None] = None,
|
643
|
+
params: Optional[Dict[str, Any]] = None,
|
644
|
+
order: str = 'asc',
|
645
|
+
limit: Optional[int] = None,
|
646
|
+
begin_add_minutes: int = 0,
|
647
|
+
end_add_minutes: int = 0,
|
648
|
+
debug: bool = False,
|
649
|
+
**kw: Any
|
650
|
+
) -> Union[pd.DataFrame, None]:
|
651
651
|
"""
|
652
652
|
Access a pipe's data from the SQL instance.
|
653
653
|
|
@@ -746,19 +746,19 @@ def get_pipe_data(
|
|
746
746
|
}
|
747
747
|
query = self.get_pipe_data_query(
|
748
748
|
pipe,
|
749
|
-
select_columns
|
750
|
-
omit_columns
|
751
|
-
begin
|
752
|
-
end
|
753
|
-
params
|
754
|
-
order
|
755
|
-
limit
|
756
|
-
begin_add_minutes
|
757
|
-
end_add_minutes
|
758
|
-
debug
|
749
|
+
select_columns=select_columns,
|
750
|
+
omit_columns=omit_columns,
|
751
|
+
begin=begin,
|
752
|
+
end=end,
|
753
|
+
params=params,
|
754
|
+
order=order,
|
755
|
+
limit=limit,
|
756
|
+
begin_add_minutes=begin_add_minutes,
|
757
|
+
end_add_minutes=end_add_minutes,
|
758
|
+
debug=debug,
|
759
759
|
**kw
|
760
760
|
)
|
761
|
-
|
761
|
+
|
762
762
|
if is_dask:
|
763
763
|
index_col = pipe.columns.get('datetime', None)
|
764
764
|
kw['index_col'] = index_col
|
@@ -769,11 +769,11 @@ def get_pipe_data(
|
|
769
769
|
if typ == 'numeric' and col in dtypes
|
770
770
|
]
|
771
771
|
kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
|
772
|
-
|
772
|
+
|
773
773
|
df = self.read(
|
774
774
|
query,
|
775
|
-
dtype
|
776
|
-
debug
|
775
|
+
dtype=dtypes,
|
776
|
+
debug=debug,
|
777
777
|
**kw
|
778
778
|
)
|
779
779
|
for col in numeric_columns:
|
@@ -782,28 +782,25 @@ def get_pipe_data(
|
|
782
782
|
df[col] = df[col].apply(attempt_cast_to_numeric)
|
783
783
|
|
784
784
|
if self.flavor == 'sqlite':
|
785
|
+
ignore_dt_cols = [
|
786
|
+
col
|
787
|
+
for col, dtype in pipe.dtypes.items()
|
788
|
+
if 'datetime' not in str(dtype)
|
789
|
+
]
|
785
790
|
### NOTE: We have to consume the iterator here to ensure that datetimes are parsed correctly
|
786
791
|
df = (
|
787
792
|
parse_df_datetimes(
|
788
793
|
df,
|
789
|
-
ignore_cols
|
790
|
-
|
791
|
-
|
792
|
-
if 'datetime' not in str(dtype)
|
793
|
-
],
|
794
|
-
chunksize = kw.get('chunksize', None),
|
795
|
-
debug = debug,
|
794
|
+
ignore_cols=ignore_dt_cols,
|
795
|
+
chunksize=kw.get('chunksize', None),
|
796
|
+
debug=debug,
|
796
797
|
) if isinstance(df, pd.DataFrame) else (
|
797
798
|
[
|
798
799
|
parse_df_datetimes(
|
799
800
|
c,
|
800
|
-
ignore_cols
|
801
|
-
col
|
802
|
-
for col, dtype in pipe.dtypes.items()
|
803
|
-
if 'datetime' not in str(dtype)
|
804
|
-
],
|
801
|
+
ignore_cols=ignore_dt_cols,
|
805
802
|
chunksize = kw.get('chunksize', None),
|
806
|
-
debug
|
803
|
+
debug=debug,
|
807
804
|
)
|
808
805
|
for c in df
|
809
806
|
]
|
@@ -817,21 +814,22 @@ def get_pipe_data(
|
|
817
814
|
|
818
815
|
|
819
816
|
def get_pipe_data_query(
|
820
|
-
|
821
|
-
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
830
|
-
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
817
|
+
self,
|
818
|
+
pipe: mrsm.Pipe,
|
819
|
+
select_columns: Optional[List[str]] = None,
|
820
|
+
omit_columns: Optional[List[str]] = None,
|
821
|
+
begin: Union[datetime, int, str, None] = None,
|
822
|
+
end: Union[datetime, int, str, None] = None,
|
823
|
+
params: Optional[Dict[str, Any]] = None,
|
824
|
+
order: Optional[str] = 'asc',
|
825
|
+
sort_datetimes: bool = False,
|
826
|
+
limit: Optional[int] = None,
|
827
|
+
begin_add_minutes: int = 0,
|
828
|
+
end_add_minutes: int = 0,
|
829
|
+
replace_nulls: Optional[str] = None,
|
830
|
+
debug: bool = False,
|
831
|
+
**kw: Any
|
832
|
+
) -> Union[str, None]:
|
835
833
|
"""
|
836
834
|
Return the `SELECT` query for retrieving a pipe's data from its instance.
|
837
835
|
|
@@ -857,10 +855,13 @@ def get_pipe_data_query(
|
|
857
855
|
Additional parameters to filter by.
|
858
856
|
See `meerschaum.connectors.sql.build_where`.
|
859
857
|
|
860
|
-
order: Optional[str], default
|
858
|
+
order: Optional[str], default None
|
861
859
|
The selection order for all of the indices in the query.
|
862
860
|
If `None`, omit the `ORDER BY` clause.
|
863
861
|
|
862
|
+
sort_datetimes: bool, default False
|
863
|
+
Alias for `order='desc'`.
|
864
|
+
|
864
865
|
limit: Optional[int], default None
|
865
866
|
If specified, limit the number of rows retrieved to this value.
|
866
867
|
|
@@ -883,7 +884,6 @@ def get_pipe_data_query(
|
|
883
884
|
-------
|
884
885
|
A `SELECT` query to retrieve a pipe's data.
|
885
886
|
"""
|
886
|
-
import json
|
887
887
|
from meerschaum.utils.debug import dprint
|
888
888
|
from meerschaum.utils.misc import items_str
|
889
889
|
from meerschaum.utils.sql import sql_item_name, dateadd_str
|
@@ -898,6 +898,9 @@ def get_pipe_data_query(
|
|
898
898
|
if omit_columns:
|
899
899
|
select_columns = [col for col in select_columns if col not in omit_columns]
|
900
900
|
|
901
|
+
if order is None and sort_datetimes:
|
902
|
+
order = 'desc'
|
903
|
+
|
901
904
|
if begin == '':
|
902
905
|
begin = pipe.get_sync_time(debug=debug)
|
903
906
|
backtrack_interval = pipe.get_backtrack_interval(debug=debug)
|
@@ -906,7 +909,7 @@ def get_pipe_data_query(
|
|
906
909
|
|
907
910
|
cols_names = [sql_item_name(col, self.flavor, None) for col in select_columns]
|
908
911
|
select_cols_str = (
|
909
|
-
'SELECT\n'
|
912
|
+
'SELECT\n '
|
910
913
|
+ ',\n '.join(
|
911
914
|
[
|
912
915
|
(
|
@@ -950,23 +953,23 @@ def get_pipe_data_query(
|
|
950
953
|
warn(
|
951
954
|
f"No datetime could be determined for {pipe}."
|
952
955
|
+ "\n Ignoring begin and end...",
|
953
|
-
stack
|
956
|
+
stack=False,
|
954
957
|
)
|
955
958
|
begin, end = None, None
|
956
959
|
else:
|
957
960
|
warn(
|
958
961
|
f"A datetime wasn't specified for {pipe}.\n"
|
959
962
|
+ f" Using column \"{_dt}\" for datetime bounds...",
|
960
|
-
stack
|
963
|
+
stack=False,
|
961
964
|
)
|
962
965
|
|
963
966
|
is_dt_bound = False
|
964
967
|
if begin is not None and _dt in existing_cols:
|
965
968
|
begin_da = dateadd_str(
|
966
|
-
flavor
|
967
|
-
datepart
|
968
|
-
number
|
969
|
-
begin
|
969
|
+
flavor=self.flavor,
|
970
|
+
datepart='minute',
|
971
|
+
number=begin_add_minutes,
|
972
|
+
begin=begin,
|
970
973
|
)
|
971
974
|
where += f"{dt} >= {begin_da}" + (" AND " if end is not None else "")
|
972
975
|
is_dt_bound = True
|
@@ -975,10 +978,10 @@ def get_pipe_data_query(
|
|
975
978
|
if 'int' in str(type(end)).lower() and end == begin:
|
976
979
|
end += 1
|
977
980
|
end_da = dateadd_str(
|
978
|
-
flavor
|
979
|
-
datepart
|
980
|
-
number
|
981
|
-
begin
|
981
|
+
flavor=self.flavor,
|
982
|
+
datepart='minute',
|
983
|
+
number=end_add_minutes,
|
984
|
+
begin=end
|
982
985
|
)
|
983
986
|
where += f"{dt} < {end_da}"
|
984
987
|
is_dt_bound = True
|
@@ -1011,9 +1014,12 @@ def get_pipe_data_query(
|
|
1011
1014
|
|
1012
1015
|
if isinstance(limit, int):
|
1013
1016
|
if self.flavor == 'mssql':
|
1014
|
-
query = f'SELECT TOP {limit}
|
1017
|
+
query = f'SELECT TOP {limit}\n' + query[len("SELECT "):]
|
1015
1018
|
elif self.flavor == 'oracle':
|
1016
|
-
query =
|
1019
|
+
query = (
|
1020
|
+
f"SELECT * FROM (\n {query}\n)\n"
|
1021
|
+
+ f"WHERE ROWNUM IN ({', '.join([str(i) for i in range(1, limit+1)])})"
|
1022
|
+
)
|
1017
1023
|
else:
|
1018
1024
|
query += f"\nLIMIT {limit}"
|
1019
1025
|
|
@@ -1107,18 +1113,18 @@ def get_pipe_attributes(
|
|
1107
1113
|
|
1108
1114
|
|
1109
1115
|
def sync_pipe(
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
1113
|
-
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1116
|
+
self,
|
1117
|
+
pipe: mrsm.Pipe,
|
1118
|
+
df: Union[pd.DataFrame, str, Dict[Any, Any], None] = None,
|
1119
|
+
begin: Optional[datetime] = None,
|
1120
|
+
end: Optional[datetime] = None,
|
1121
|
+
chunksize: Optional[int] = -1,
|
1122
|
+
check_existing: bool = True,
|
1123
|
+
blocking: bool = True,
|
1124
|
+
debug: bool = False,
|
1125
|
+
_check_temporary_tables: bool = True,
|
1126
|
+
**kw: Any
|
1127
|
+
) -> SuccessTuple:
|
1122
1128
|
"""
|
1123
1129
|
Sync a pipe using a database connection.
|
1124
1130
|
|
@@ -1191,9 +1197,9 @@ def sync_pipe(
|
|
1191
1197
|
if not isinstance(df, pd.DataFrame):
|
1192
1198
|
df = pipe.enforce_dtypes(
|
1193
1199
|
df,
|
1194
|
-
chunksize
|
1195
|
-
safe_copy
|
1196
|
-
debug
|
1200
|
+
chunksize=chunksize,
|
1201
|
+
safe_copy=kw.get('safe_copy', False),
|
1202
|
+
debug=debug,
|
1197
1203
|
)
|
1198
1204
|
|
1199
1205
|
### if table does not exist, create it with indices
|
@@ -1243,8 +1249,8 @@ def sync_pipe(
|
|
1243
1249
|
unseen_df, update_df, delta_df = (
|
1244
1250
|
pipe.filter_existing(
|
1245
1251
|
df,
|
1246
|
-
chunksize
|
1247
|
-
debug
|
1252
|
+
chunksize=chunksize,
|
1253
|
+
debug=debug,
|
1248
1254
|
**kw
|
1249
1255
|
) if check_existing else (df, None, df)
|
1250
1256
|
)
|
@@ -1263,31 +1269,6 @@ def sync_pipe(
|
|
1263
1269
|
if 'name' in kw:
|
1264
1270
|
kw.pop('name')
|
1265
1271
|
|
1266
|
-
### Account for first-time syncs of JSON columns.
|
1267
|
-
unseen_json_cols = get_json_cols(unseen_df)
|
1268
|
-
update_json_cols = get_json_cols(update_df) if update_df is not None else []
|
1269
|
-
json_cols = list(set(unseen_json_cols + update_json_cols))
|
1270
|
-
existing_json_cols = [col for col, typ in pipe.dtypes.items() if typ == 'json']
|
1271
|
-
new_json_cols = [col for col in json_cols if col not in existing_json_cols]
|
1272
|
-
if new_json_cols:
|
1273
|
-
pipe.dtypes.update({col: 'json' for col in json_cols})
|
1274
|
-
if not pipe.temporary:
|
1275
|
-
edit_success, edit_msg = pipe.edit(interactive=False, debug=debug)
|
1276
|
-
if not edit_success:
|
1277
|
-
warn(f"Unable to update JSON dtypes for {pipe}:\n{edit_msg}")
|
1278
|
-
|
1279
|
-
unseen_numeric_cols = get_numeric_cols(unseen_df)
|
1280
|
-
update_numeric_cols = get_numeric_cols(update_df) if update_df is not None else []
|
1281
|
-
numeric_cols = list(set(unseen_numeric_cols + update_numeric_cols))
|
1282
|
-
existing_numeric_cols = [col for col, typ in pipe.dtypes.items() if typ == 'numeric']
|
1283
|
-
new_numeric_cols = [col for col in numeric_cols if col not in existing_numeric_cols]
|
1284
|
-
if new_numeric_cols:
|
1285
|
-
pipe.dtypes.update({col: 'numeric' for col in numeric_cols})
|
1286
|
-
if not pipe.temporary:
|
1287
|
-
edit_success, edit_msg = pipe.edit(interactive=False, debug=debug)
|
1288
|
-
if not edit_success:
|
1289
|
-
warn(f"Unable to update NUMERIC dtypes for {pipe}:\n{edit_msg}")
|
1290
|
-
|
1291
1272
|
### Insert new data into Pipe's table.
|
1292
1273
|
unseen_kw = copy.deepcopy(kw)
|
1293
1274
|
unseen_kw.update({
|
@@ -1326,16 +1307,16 @@ def sync_pipe(
|
|
1326
1307
|
self._log_temporary_tables_creation(temp_target, create=(not pipe.temporary), debug=debug)
|
1327
1308
|
temp_pipe = Pipe(
|
1328
1309
|
pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
|
1329
|
-
instance
|
1330
|
-
columns
|
1310
|
+
instance=pipe.instance_keys,
|
1311
|
+
columns={
|
1331
1312
|
ix_key: ix
|
1332
1313
|
for ix_key, ix in pipe.columns.items()
|
1333
1314
|
if ix and ix in update_df.columns
|
1334
1315
|
},
|
1335
|
-
dtypes
|
1336
|
-
target
|
1337
|
-
temporary
|
1338
|
-
parameters
|
1316
|
+
dtypes=pipe.dtypes,
|
1317
|
+
target=temp_target,
|
1318
|
+
temporary=True,
|
1319
|
+
parameters={
|
1339
1320
|
'schema': self.internal_schema,
|
1340
1321
|
'hypertable': False,
|
1341
1322
|
},
|
@@ -1352,20 +1333,20 @@ def sync_pipe(
|
|
1352
1333
|
temp_target,
|
1353
1334
|
self,
|
1354
1335
|
join_cols,
|
1355
|
-
upsert
|
1356
|
-
schema
|
1357
|
-
patch_schema
|
1358
|
-
datetime_col
|
1359
|
-
debug
|
1336
|
+
upsert=upsert,
|
1337
|
+
schema=self.get_pipe_schema(pipe),
|
1338
|
+
patch_schema=self.internal_schema,
|
1339
|
+
datetime_col=pipe.columns.get('datetime', None),
|
1340
|
+
debug=debug,
|
1360
1341
|
)
|
1361
1342
|
update_success = all(
|
1362
1343
|
self.exec_queries(update_queries, break_on_error=True, rollback=True, debug=debug)
|
1363
1344
|
)
|
1364
1345
|
self._log_temporary_tables_creation(
|
1365
1346
|
temp_target,
|
1366
|
-
ready_to_drop
|
1367
|
-
create
|
1368
|
-
debug
|
1347
|
+
ready_to_drop=True,
|
1348
|
+
create=(not pipe.temporary),
|
1349
|
+
debug=debug,
|
1369
1350
|
)
|
1370
1351
|
if not update_success:
|
1371
1352
|
warn(f"Failed to apply update to {pipe}.")
|
@@ -1406,16 +1387,16 @@ def sync_pipe(
|
|
1406
1387
|
|
1407
1388
|
|
1408
1389
|
def sync_pipe_inplace(
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1412
|
-
|
1413
|
-
|
1414
|
-
|
1415
|
-
|
1416
|
-
|
1417
|
-
|
1418
|
-
|
1390
|
+
self,
|
1391
|
+
pipe: 'mrsm.Pipe',
|
1392
|
+
params: Optional[Dict[str, Any]] = None,
|
1393
|
+
begin: Optional[datetime] = None,
|
1394
|
+
end: Optional[datetime] = None,
|
1395
|
+
chunksize: Optional[int] = -1,
|
1396
|
+
check_existing: bool = True,
|
1397
|
+
debug: bool = False,
|
1398
|
+
**kw: Any
|
1399
|
+
) -> SuccessTuple:
|
1419
1400
|
"""
|
1420
1401
|
If a pipe's connector is the same as its instance connector,
|
1421
1402
|
it's more efficient to sync the pipe in-place rather than reading data into Pandas.
|
@@ -1455,13 +1436,13 @@ def sync_pipe_inplace(
|
|
1455
1436
|
"""
|
1456
1437
|
if self.flavor == 'duckdb':
|
1457
1438
|
return pipe.sync(
|
1458
|
-
params
|
1459
|
-
begin
|
1460
|
-
end
|
1461
|
-
chunksize
|
1462
|
-
check_existing
|
1463
|
-
debug
|
1464
|
-
_inplace
|
1439
|
+
params=params,
|
1440
|
+
begin=begin,
|
1441
|
+
end=end,
|
1442
|
+
chunksize=chunksize,
|
1443
|
+
check_existing=check_existing,
|
1444
|
+
debug=debug,
|
1445
|
+
_inplace=False,
|
1465
1446
|
**kw
|
1466
1447
|
)
|
1467
1448
|
from meerschaum.utils.sql import (
|
@@ -1627,45 +1608,39 @@ def sync_pipe_inplace(
|
|
1627
1608
|
|
1628
1609
|
backtrack_def = self.get_pipe_data_query(
|
1629
1610
|
pipe,
|
1630
|
-
begin
|
1631
|
-
end
|
1632
|
-
begin_add_minutes
|
1633
|
-
end_add_minutes
|
1634
|
-
params
|
1635
|
-
debug
|
1636
|
-
order
|
1611
|
+
begin=begin,
|
1612
|
+
end=end,
|
1613
|
+
begin_add_minutes=0,
|
1614
|
+
end_add_minutes=1,
|
1615
|
+
params=params,
|
1616
|
+
debug=debug,
|
1617
|
+
order=None,
|
1637
1618
|
)
|
1638
1619
|
|
1639
|
-
select_backtrack_query = format_cte_subquery(
|
1640
|
-
backtrack_def,
|
1641
|
-
self.flavor,
|
1642
|
-
sub_name = 'backtrack_def',
|
1643
|
-
)
|
1644
1620
|
create_backtrack_query = get_create_table_query(
|
1645
1621
|
backtrack_def,
|
1646
1622
|
temp_tables['backtrack'],
|
1647
1623
|
self.flavor,
|
1648
|
-
schema
|
1624
|
+
schema=internal_schema,
|
1649
1625
|
)
|
1650
1626
|
(create_backtrack_success, create_backtrack_msg), create_backtrack_results = session_execute(
|
1651
1627
|
session,
|
1652
1628
|
create_backtrack_query,
|
1653
|
-
with_results
|
1654
|
-
debug
|
1629
|
+
with_results=True,
|
1630
|
+
debug=debug,
|
1655
1631
|
) if not upsert else (True, "Success"), None
|
1656
1632
|
|
1657
1633
|
if not create_backtrack_success:
|
1658
1634
|
_ = clean_up_temp_tables()
|
1659
1635
|
return create_backtrack_success, create_backtrack_msg
|
1660
|
-
bactrack_count = create_backtrack_results[0].rowcount if create_backtrack_results else 0
|
1661
1636
|
|
1662
1637
|
backtrack_cols_types = get_table_cols_types(
|
1663
1638
|
temp_tables['backtrack'],
|
1664
|
-
connectable
|
1665
|
-
flavor
|
1666
|
-
schema
|
1667
|
-
database
|
1668
|
-
debug
|
1639
|
+
connectable=connectable,
|
1640
|
+
flavor=self.flavor,
|
1641
|
+
schema=internal_schema,
|
1642
|
+
database=database,
|
1643
|
+
debug=debug,
|
1669
1644
|
) if not upsert else new_cols_types
|
1670
1645
|
|
1671
1646
|
common_cols = [col for col in new_cols if col in backtrack_cols_types]
|
@@ -1691,7 +1666,7 @@ def sync_pipe_inplace(
|
|
1691
1666
|
)
|
1692
1667
|
|
1693
1668
|
select_delta_query = (
|
1694
|
-
|
1669
|
+
"SELECT\n"
|
1695
1670
|
+ null_replace_new_cols_str + "\n"
|
1696
1671
|
+ f"\nFROM {temp_table_names['new']}\n"
|
1697
1672
|
+ f"LEFT OUTER JOIN {temp_table_names['backtrack']}\nON\n"
|
@@ -1721,12 +1696,12 @@ def sync_pipe_inplace(
|
|
1721
1696
|
select_delta_query,
|
1722
1697
|
temp_tables['delta'],
|
1723
1698
|
self.flavor,
|
1724
|
-
schema
|
1699
|
+
schema=internal_schema,
|
1725
1700
|
)
|
1726
1701
|
create_delta_success, create_delta_msg = session_execute(
|
1727
1702
|
session,
|
1728
1703
|
create_delta_query,
|
1729
|
-
debug
|
1704
|
+
debug=debug,
|
1730
1705
|
) if not upsert else (True, "Success")
|
1731
1706
|
if not create_delta_success:
|
1732
1707
|
_ = clean_up_temp_tables()
|
@@ -1735,10 +1710,10 @@ def sync_pipe_inplace(
|
|
1735
1710
|
delta_cols_types = get_table_cols_types(
|
1736
1711
|
temp_tables['delta'],
|
1737
1712
|
connectable = connectable,
|
1738
|
-
flavor
|
1739
|
-
schema
|
1740
|
-
database
|
1741
|
-
debug
|
1713
|
+
flavor=self.flavor,
|
1714
|
+
schema=internal_schema,
|
1715
|
+
database=database,
|
1716
|
+
debug=debug,
|
1742
1717
|
) if not upsert else new_cols_types
|
1743
1718
|
|
1744
1719
|
### This is a weird bug on SQLite.
|
@@ -2062,14 +2037,14 @@ def pipe_exists(
|
|
2062
2037
|
|
2063
2038
|
|
2064
2039
|
def get_pipe_rowcount(
|
2065
|
-
|
2066
|
-
|
2067
|
-
|
2068
|
-
|
2069
|
-
|
2070
|
-
|
2071
|
-
|
2072
|
-
|
2040
|
+
self,
|
2041
|
+
pipe: mrsm.Pipe,
|
2042
|
+
begin: Union[datetime, int, None] = None,
|
2043
|
+
end: Union[datetime, int, None] = None,
|
2044
|
+
params: Optional[Dict[str, Any]] = None,
|
2045
|
+
remote: bool = False,
|
2046
|
+
debug: bool = False
|
2047
|
+
) -> Union[int, None]:
|
2073
2048
|
"""
|
2074
2049
|
Get the rowcount for a pipe in accordance with given parameters.
|
2075
2050
|
|
@@ -2411,11 +2386,11 @@ def get_pipe_columns_types(
|
|
2411
2386
|
|
2412
2387
|
|
2413
2388
|
def get_add_columns_queries(
|
2414
|
-
|
2415
|
-
|
2416
|
-
|
2417
|
-
|
2418
|
-
|
2389
|
+
self,
|
2390
|
+
pipe: mrsm.Pipe,
|
2391
|
+
df: Union[pd.DataFrame, Dict[str, str]],
|
2392
|
+
debug: bool = False,
|
2393
|
+
) -> List[str]:
|
2419
2394
|
"""
|
2420
2395
|
Add new null columns of the correct type to a table from a dataframe.
|
2421
2396
|
|