FlowerPower 0.11.6.6__tar.gz → 0.11.6.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flowerpower-0.11.6.6/src/FlowerPower.egg-info → flowerpower-0.11.6.8}/PKG-INFO +1 -1
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/pyproject.toml +1 -1
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8/src/FlowerPower.egg-info}/PKG-INFO +1 -1
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/fs/ext.py +77 -47
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/utils/misc.py +50 -23
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/LICENSE +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/README.md +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/setup.cfg +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/FlowerPower.egg-info/SOURCES.txt +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/FlowerPower.egg-info/dependency_links.txt +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/FlowerPower.egg-info/entry_points.txt +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/FlowerPower.egg-info/requires.txt +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/FlowerPower.egg-info/top_level.txt +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/base.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/pipeline/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/pipeline/adapter.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/pipeline/run.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/pipeline/schedule.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/project/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/project/adapter.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cfg/project/job_queue.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cli/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cli/cfg.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cli/job_queue.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cli/mqtt.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cli/pipeline.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/cli/utils.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/flowerpower.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/fs/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/fs/base.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/fs/storage_options.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/manager.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/setup.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/trigger.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/utils.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/base.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/rq/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/rq/_trigger.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/rq/manager.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/rq/setup.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/rq/utils.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/mqtt.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/pipeline/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/pipeline/base.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/pipeline/io.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/pipeline/job_queue.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/pipeline/manager.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/pipeline/registry.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/pipeline/runner.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/pipeline/visualizer.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/base.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/helpers/datetime.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/helpers/polars.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/helpers/pyarrow.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/helpers/sql.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/csv.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/deltatable.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/duckdb.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/json.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/mqtt.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/mssql.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/mysql.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/oracle.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/parquet.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/postgres.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/pydala.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/sqlite.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/metadata.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/csv.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/deltatable.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/duckdb.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/json.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/mqtt.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/mssql.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/mysql.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/oracle.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/parquet.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/postgres.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/pydala.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/sqlite.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/mqtt/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/mqtt/cfg.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/mqtt/manager.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/settings/__init__.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/settings/backend.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/settings/executor.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/settings/general.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/settings/hamilton.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/settings/job_queue.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/settings/logging.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/settings/retry.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/utils/callback.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/utils/logging.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/utils/monkey.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/utils/open_telemetry.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/utils/scheduler.py +0 -0
- {flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/utils/templates.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: FlowerPower
|
3
|
-
Version: 0.11.6.
|
3
|
+
Version: 0.11.6.8
|
4
4
|
Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
|
5
5
|
Author-email: "Volker L." <ligno.blades@gmail.com>
|
6
6
|
Project-URL: Homepage, https://github.com/legout/flowerpower
|
@@ -4,7 +4,7 @@ description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower
|
|
4
4
|
authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
|
5
5
|
readme = "README.md"
|
6
6
|
requires-python = ">= 3.11"
|
7
|
-
version = "0.11.6.
|
7
|
+
version = "0.11.6.8"
|
8
8
|
keywords = [
|
9
9
|
"hamilton",
|
10
10
|
"workflow",
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: FlowerPower
|
3
|
-
Version: 0.11.6.
|
3
|
+
Version: 0.11.6.8
|
4
4
|
Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
|
5
5
|
Author-email: "Volker L." <ligno.blades@gmail.com>
|
6
6
|
Project-URL: Homepage, https://github.com/legout/flowerpower
|
@@ -814,6 +814,7 @@ def _read_parquet_file(
|
|
814
814
|
path: Path to Parquet file
|
815
815
|
self: Filesystem instance to use for reading
|
816
816
|
include_file_path: Add source filepath as a column
|
817
|
+
opt_dtypes: Optimize DataFrame dtypes
|
817
818
|
**kwargs: Additional arguments passed to pq.read_table()
|
818
819
|
|
819
820
|
Returns:
|
@@ -830,6 +831,11 @@ def _read_parquet_file(
|
|
830
831
|
>>> print("file_path" in table.column_names)
|
831
832
|
True
|
832
833
|
"""
|
834
|
+
if not path.endswith(".parquet"):
|
835
|
+
raise ValueError(
|
836
|
+
f"Path '{path}' does not point to a Parquet file. "
|
837
|
+
"Ensure the path ends with '.parquet'."
|
838
|
+
)
|
833
839
|
table = pq.read_table(path, filesystem=self, **kwargs)
|
834
840
|
if include_file_path:
|
835
841
|
table = table.add_column(0, "file_path", pl.Series([path] * table.num_rows))
|
@@ -841,6 +847,30 @@ def _read_parquet_file(
|
|
841
847
|
def read_parquet_file(
|
842
848
|
self, path: str, include_file_path: bool = False, opt_dtypes: bool = False, **kwargs
|
843
849
|
) -> pa.Table:
|
850
|
+
"""Read a single Parquet file from any filesystem.
|
851
|
+
|
852
|
+
Internal function that handles reading individual Parquet files and
|
853
|
+
optionally adds the source filepath as a column.
|
854
|
+
|
855
|
+
Args:
|
856
|
+
path: Path to Parquet file
|
857
|
+
include_file_path: Add source filepath as a column
|
858
|
+
opt_dtypes: Optimize DataFrame dtypes
|
859
|
+
**kwargs: Additional arguments passed to pq.read_table()
|
860
|
+
|
861
|
+
Returns:
|
862
|
+
pa.Table: PyArrow Table containing Parquet data
|
863
|
+
|
864
|
+
Example:
|
865
|
+
>>> fs = LocalFileSystem()
|
866
|
+
>>> table = fs.read_parquet_file(
|
867
|
+
... "data.parquet",
|
868
|
+
... include_file_path=True,
|
869
|
+
... use_threads=True
|
870
|
+
... )
|
871
|
+
>>> print("file_path" in table.column_names)
|
872
|
+
True
|
873
|
+
"""
|
844
874
|
return _read_parquet_file(
|
845
875
|
path=path,
|
846
876
|
self=self,
|
@@ -852,7 +882,7 @@ def read_parquet_file(
|
|
852
882
|
|
853
883
|
def _read_parquet(
|
854
884
|
self,
|
855
|
-
path,
|
885
|
+
path: str | list[str],
|
856
886
|
include_file_path: bool = False,
|
857
887
|
use_threads: bool = True,
|
858
888
|
concat: bool = True,
|
@@ -874,48 +904,48 @@ def _read_parquet(
|
|
874
904
|
Returns:
|
875
905
|
(pa.Table | list[pa.Table]): Pyarrow Table or list of Pyarrow Tables.
|
876
906
|
"""
|
877
|
-
if not include_file_path and concat:
|
878
|
-
|
879
|
-
|
880
|
-
|
881
|
-
|
882
|
-
else:
|
883
|
-
|
884
|
-
|
885
|
-
|
907
|
+
# if not include_file_path and concat:
|
908
|
+
# if isinstance(path, str):
|
909
|
+
# path = path.replace("**", "").replace("*.parquet", "")
|
910
|
+
# table = _read_parquet_file(path, self=self, opt_dtypes=opt_dtypes, **kwargs)
|
911
|
+
# return table
|
912
|
+
# else:
|
913
|
+
if isinstance(path, str):
|
914
|
+
path = path_to_glob(path, format="parquet")
|
915
|
+
path = self.glob(path)
|
886
916
|
|
887
|
-
|
888
|
-
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
self=self,
|
893
|
-
include_file_path=include_file_path,
|
894
|
-
opt_dtypes=opt_dtypes,
|
895
|
-
n_jobs=-1,
|
896
|
-
backend="threading",
|
897
|
-
verbose=verbose,
|
898
|
-
**kwargs,
|
899
|
-
)
|
900
|
-
else:
|
901
|
-
tables = [
|
902
|
-
_read_parquet_file(
|
903
|
-
p,
|
904
|
-
self=self,
|
905
|
-
include_file_path=include_file_path,
|
906
|
-
opt_dtypes=opt_dtypes,
|
907
|
-
**kwargs,
|
908
|
-
)
|
909
|
-
for p in path
|
910
|
-
]
|
911
|
-
else:
|
912
|
-
tables = _read_parquet_file(
|
913
|
-
path=path,
|
917
|
+
if isinstance(path, list):
|
918
|
+
if use_threads:
|
919
|
+
tables = run_parallel(
|
920
|
+
_read_parquet_file,
|
921
|
+
path,
|
914
922
|
self=self,
|
915
923
|
include_file_path=include_file_path,
|
916
924
|
opt_dtypes=opt_dtypes,
|
925
|
+
n_jobs=-1,
|
926
|
+
backend="threading",
|
927
|
+
verbose=verbose,
|
917
928
|
**kwargs,
|
918
929
|
)
|
930
|
+
else:
|
931
|
+
tables = [
|
932
|
+
_read_parquet_file(
|
933
|
+
p,
|
934
|
+
self=self,
|
935
|
+
include_file_path=include_file_path,
|
936
|
+
opt_dtypes=opt_dtypes,
|
937
|
+
**kwargs,
|
938
|
+
)
|
939
|
+
for p in path
|
940
|
+
]
|
941
|
+
else:
|
942
|
+
tables = _read_parquet_file(
|
943
|
+
path=path,
|
944
|
+
self=self,
|
945
|
+
include_file_path=include_file_path,
|
946
|
+
opt_dtypes=opt_dtypes,
|
947
|
+
**kwargs,
|
948
|
+
)
|
919
949
|
if concat:
|
920
950
|
# Unify schemas before concatenation if opt_dtypes or multiple tables
|
921
951
|
if isinstance(tables, list):
|
@@ -1001,14 +1031,14 @@ def _read_parquet_batches(
|
|
1001
1031
|
... print(f"Batch schema: {batch.schema}")
|
1002
1032
|
"""
|
1003
1033
|
# Fast path for simple cases
|
1004
|
-
if not include_file_path and concat and batch_size is None:
|
1005
|
-
|
1006
|
-
|
1007
|
-
|
1008
|
-
|
1009
|
-
|
1010
|
-
|
1011
|
-
|
1034
|
+
# if not include_file_path and concat and batch_size is None:
|
1035
|
+
# if isinstance(path, str):
|
1036
|
+
# path = path.replace("**", "").replace("*.parquet", "")
|
1037
|
+
# table = _read_parquet_file(
|
1038
|
+
# path=path, self=self, opt_dtypes=opt_dtypes, **kwargs
|
1039
|
+
# )
|
1040
|
+
# yield table
|
1041
|
+
# return
|
1012
1042
|
|
1013
1043
|
# Resolve path(s) to list
|
1014
1044
|
if isinstance(path, str):
|
@@ -1058,13 +1088,13 @@ def _read_parquet_batches(
|
|
1058
1088
|
schemas = [t.schema for t in batch_tables]
|
1059
1089
|
unified_schema = unify_schemas_pa(schemas)
|
1060
1090
|
batch_tables = [cast_schema(t, unified_schema) for t in batch_tables]
|
1061
|
-
|
1091
|
+
batch_table = pa.concat_tables(
|
1062
1092
|
[table for table in batch_tables if table.num_rows > 0],
|
1063
1093
|
promote_options="permissive",
|
1064
1094
|
)
|
1065
1095
|
# if opt_dtypes:
|
1066
1096
|
# result = opt_dtype_pa(result, strict=False)
|
1067
|
-
yield
|
1097
|
+
yield batch_table
|
1068
1098
|
else:
|
1069
1099
|
# if opt_dtypes and isinstance(batch_tables, list):
|
1070
1100
|
# batch_tables = [opt_dtype_pa(t, strict=False) for t in batch_tables]
|
@@ -266,6 +266,8 @@ else:
|
|
266
266
|
|
267
267
|
if importlib.util.find_spec("joblib"):
|
268
268
|
from joblib import Parallel, delayed
|
269
|
+
from rich.progress import (BarColumn, Progress, TextColumn,
|
270
|
+
TimeElapsedColumn)
|
269
271
|
|
270
272
|
def run_parallel(
|
271
273
|
func: callable,
|
@@ -299,19 +301,15 @@ if importlib.util.find_spec("joblib"):
|
|
299
301
|
>>> # Only kwargs iterables
|
300
302
|
>>> run_parallel(func, x=[1,2,3], y=[4,5,6], fixed=42)
|
301
303
|
"""
|
302
|
-
# Special kwargs for run_parallel itself
|
303
304
|
parallel_kwargs = {"n_jobs": n_jobs, "backend": backend, "verbose": 0}
|
304
305
|
|
305
|
-
# Process args and kwargs to separate iterables and fixed values
|
306
306
|
iterables = []
|
307
307
|
fixed_args = []
|
308
308
|
iterable_kwargs = {}
|
309
309
|
fixed_kwargs = {}
|
310
310
|
|
311
|
-
# Get the length of the first iterable to determine number of iterations
|
312
311
|
first_iterable_len = None
|
313
312
|
|
314
|
-
# Process args
|
315
313
|
for arg in args:
|
316
314
|
if isinstance(arg, (list, tuple)) and not isinstance(arg[0], (list, tuple)):
|
317
315
|
iterables.append(arg)
|
@@ -324,7 +322,6 @@ if importlib.util.find_spec("joblib"):
|
|
324
322
|
else:
|
325
323
|
fixed_args.append(arg)
|
326
324
|
|
327
|
-
# Process kwargs
|
328
325
|
for key, value in kwargs.items():
|
329
326
|
if isinstance(value, (list, tuple)) and not isinstance(
|
330
327
|
value[0], (list, tuple)
|
@@ -342,26 +339,56 @@ if importlib.util.find_spec("joblib"):
|
|
342
339
|
if first_iterable_len is None:
|
343
340
|
raise ValueError("At least one iterable argument is required")
|
344
341
|
|
345
|
-
# Create parameter combinations
|
346
342
|
all_iterables = iterables + list(iterable_kwargs.values())
|
347
|
-
param_combinations = list(zip(*all_iterables))
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
343
|
+
param_combinations = list(zip(*all_iterables))
|
344
|
+
|
345
|
+
if not verbose:
|
346
|
+
return Parallel(**parallel_kwargs)(
|
347
|
+
delayed(func)(
|
348
|
+
*(list(param_tuple[: len(iterables)]) + fixed_args),
|
349
|
+
**{
|
350
|
+
k: v
|
351
|
+
for k, v in zip(
|
352
|
+
iterable_kwargs.keys(), param_tuple[len(iterables) :]
|
353
|
+
)
|
354
|
+
},
|
355
|
+
**fixed_kwargs,
|
356
|
+
)
|
357
|
+
for param_tuple in param_combinations
|
362
358
|
)
|
363
|
-
|
364
|
-
|
359
|
+
else:
|
360
|
+
results = [None] * len(param_combinations)
|
361
|
+
with Progress(
|
362
|
+
TextColumn("[progress.description]{task.description}"),
|
363
|
+
BarColumn(),
|
364
|
+
"[progress.percentage]{task.percentage:>3.0f}%",
|
365
|
+
TimeElapsedColumn(),
|
366
|
+
transient=True,
|
367
|
+
) as progress:
|
368
|
+
task = progress.add_task(
|
369
|
+
"Running in parallel...", total=len(param_combinations)
|
370
|
+
)
|
371
|
+
|
372
|
+
def wrapper(idx, param_tuple):
|
373
|
+
res = func(
|
374
|
+
*(list(param_tuple[: len(iterables)]) + fixed_args),
|
375
|
+
**{
|
376
|
+
k: v
|
377
|
+
for k, v in zip(
|
378
|
+
iterable_kwargs.keys(), param_tuple[len(iterables) :]
|
379
|
+
)
|
380
|
+
},
|
381
|
+
**fixed_kwargs,
|
382
|
+
)
|
383
|
+
progress.update(task, advance=1)
|
384
|
+
return idx, res
|
385
|
+
|
386
|
+
for idx, result in Parallel(**parallel_kwargs)(
|
387
|
+
delayed(wrapper)(i, param_tuple)
|
388
|
+
for i, param_tuple in enumerate(param_combinations)
|
389
|
+
):
|
390
|
+
results[idx] = result
|
391
|
+
return results
|
365
392
|
|
366
393
|
else:
|
367
394
|
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/__init__.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
{flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/manager.py
RENAMED
File without changes
|
{flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/setup.py
RENAMED
File without changes
|
{flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/trigger.py
RENAMED
File without changes
|
{flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/job_queue/apscheduler/utils.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/helpers/datetime.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/loader/deltatable.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{flowerpower-0.11.6.6 → flowerpower-0.11.6.8}/src/flowerpower/plugins/io/saver/deltatable.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|