FlowerPower 0.11.6.2__tar.gz → 0.11.6.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. {flowerpower-0.11.6.2/src/FlowerPower.egg-info → flowerpower-0.11.6.4}/PKG-INFO +1 -1
  2. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/pyproject.toml +2 -1
  3. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4/src/FlowerPower.egg-info}/PKG-INFO +1 -1
  4. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/base.py +14 -1
  5. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/helpers/polars.py +13 -4
  6. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/helpers/pyarrow.py +11 -3
  7. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/LICENSE +0 -0
  8. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/README.md +0 -0
  9. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/setup.cfg +0 -0
  10. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/FlowerPower.egg-info/SOURCES.txt +0 -0
  11. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/FlowerPower.egg-info/dependency_links.txt +0 -0
  12. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/FlowerPower.egg-info/entry_points.txt +0 -0
  13. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/FlowerPower.egg-info/requires.txt +0 -0
  14. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/FlowerPower.egg-info/top_level.txt +0 -0
  15. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/__init__.py +0 -0
  16. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/__init__.py +0 -0
  17. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/base.py +0 -0
  18. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/pipeline/__init__.py +0 -0
  19. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/pipeline/adapter.py +0 -0
  20. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/pipeline/run.py +0 -0
  21. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/pipeline/schedule.py +0 -0
  22. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/project/__init__.py +0 -0
  23. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/project/adapter.py +0 -0
  24. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cfg/project/job_queue.py +0 -0
  25. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cli/__init__.py +0 -0
  26. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cli/cfg.py +0 -0
  27. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cli/job_queue.py +0 -0
  28. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cli/mqtt.py +0 -0
  29. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cli/pipeline.py +0 -0
  30. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/cli/utils.py +0 -0
  31. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/flowerpower.py +0 -0
  32. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/fs/__init__.py +0 -0
  33. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/fs/base.py +0 -0
  34. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/fs/ext.py +0 -0
  35. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/fs/storage_options.py +0 -0
  36. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/__init__.py +0 -0
  37. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/apscheduler/__init__.py +0 -0
  38. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -0
  39. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -0
  40. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/apscheduler/manager.py +0 -0
  41. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/apscheduler/setup.py +0 -0
  42. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/apscheduler/trigger.py +0 -0
  43. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/apscheduler/utils.py +0 -0
  44. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/base.py +0 -0
  45. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/rq/__init__.py +0 -0
  46. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/rq/_trigger.py +0 -0
  47. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -0
  48. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -0
  49. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/rq/manager.py +0 -0
  50. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/rq/setup.py +0 -0
  51. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/job_queue/rq/utils.py +0 -0
  52. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/mqtt.py +0 -0
  53. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/pipeline/__init__.py +0 -0
  54. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/pipeline/base.py +0 -0
  55. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/pipeline/io.py +0 -0
  56. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/pipeline/job_queue.py +0 -0
  57. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/pipeline/manager.py +0 -0
  58. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/pipeline/registry.py +0 -0
  59. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/pipeline/runner.py +0 -0
  60. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/pipeline/visualizer.py +0 -0
  61. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/helpers/datetime.py +0 -0
  62. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/helpers/sql.py +0 -0
  63. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/__init__.py +0 -0
  64. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/csv.py +0 -0
  65. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/deltatable.py +0 -0
  66. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/duckdb.py +0 -0
  67. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/json.py +0 -0
  68. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/mqtt.py +0 -0
  69. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/mssql.py +0 -0
  70. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/mysql.py +0 -0
  71. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/oracle.py +0 -0
  72. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/parquet.py +0 -0
  73. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/postgres.py +0 -0
  74. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/pydala.py +0 -0
  75. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/loader/sqlite.py +0 -0
  76. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/metadata.py +0 -0
  77. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/__init__.py +0 -0
  78. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/csv.py +0 -0
  79. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/deltatable.py +0 -0
  80. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/duckdb.py +0 -0
  81. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/json.py +0 -0
  82. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/mqtt.py +0 -0
  83. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/mssql.py +0 -0
  84. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/mysql.py +0 -0
  85. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/oracle.py +0 -0
  86. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/parquet.py +0 -0
  87. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/postgres.py +0 -0
  88. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/pydala.py +0 -0
  89. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/io/saver/sqlite.py +0 -0
  90. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/mqtt/__init__.py +0 -0
  91. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/mqtt/cfg.py +0 -0
  92. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/plugins/mqtt/manager.py +0 -0
  93. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/settings/__init__.py +0 -0
  94. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/settings/backend.py +0 -0
  95. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/settings/executor.py +0 -0
  96. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/settings/general.py +0 -0
  97. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/settings/hamilton.py +0 -0
  98. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/settings/job_queue.py +0 -0
  99. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/settings/logging.py +0 -0
  100. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/settings/retry.py +0 -0
  101. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/utils/callback.py +0 -0
  102. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/utils/logging.py +0 -0
  103. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/utils/misc.py +0 -0
  104. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/utils/monkey.py +0 -0
  105. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/utils/open_telemetry.py +0 -0
  106. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/utils/scheduler.py +0 -0
  107. {flowerpower-0.11.6.2 → flowerpower-0.11.6.4}/src/flowerpower/utils/templates.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: FlowerPower
3
- Version: 0.11.6.2
3
+ Version: 0.11.6.4
4
4
  Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
5
5
  Author-email: "Volker L." <ligno.blades@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -4,7 +4,7 @@ description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower
4
4
  authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
5
5
  readme = "README.md"
6
6
  requires-python = ">= 3.11"
7
- version = "0.11.6.2"
7
+ version = "0.11.6.4"
8
8
  keywords = [
9
9
  "hamilton",
10
10
  "workflow",
@@ -14,6 +14,7 @@ keywords = [
14
14
  "dask",
15
15
  "ray",
16
16
  ]
17
+
17
18
  dependencies = [
18
19
  'dill>=0.3.8',
19
20
  'duration-parser>=1.0.1',
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: FlowerPower
3
- Version: 0.11.6.2
3
+ Version: 0.11.6.4
4
4
  Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
5
5
  Author-email: "Volker L." <ligno.blades@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -335,6 +335,7 @@ class BaseFileReader(BaseFileIO, gc=False):
335
335
  tuple[pd.DataFrame | list[pd.DataFrame], dict[str, Any]] | pd.DataFrame | list[pd.DataFrame]: Pandas
336
336
  DataFrame or list of DataFrames and optional metadata.
337
337
  """
338
+ kwargs.pop("batch_size", None)
338
339
  self._load(
339
340
  reload=reload,
340
341
  metadata=metadata,
@@ -443,6 +444,8 @@ class BaseFileReader(BaseFileIO, gc=False):
443
444
  tuple[pl.DataFrame | list[pl.DataFrame], dict[str, Any]] | pl.DataFrame | list[pl.DataFrame]: Polars
444
445
  DataFrame or list of DataFrames and optional metadata.
445
446
  """
447
+ kwargs.pop("batch_size", None)
448
+
446
449
  self._load(
447
450
  metadata=metadata,
448
451
  reload=reload,
@@ -550,6 +553,8 @@ class BaseFileReader(BaseFileIO, gc=False):
550
553
  tuple[pl.LazyFrame | list[pl.LazyFrame], dict[str, Any]] | pl.LazyFrame | list[pl.LazyFrame]: Polars
551
554
  LazyFrame or list of LazyFrames and optional metadata.
552
555
  """
556
+ kwargs.pop("batch_size", None)
557
+
553
558
  self._load(
554
559
  metadata=metadata,
555
560
  reload=reload,
@@ -663,6 +668,7 @@ class BaseFileReader(BaseFileIO, gc=False):
663
668
  | list[pl.DataFrame] | list[pl.LazyFrame], dict[str, Any]]: Polars DataFrame or LazyFrame and optional
664
669
  metadata.
665
670
  """
671
+ kwargs.pop("batch_size", None)
666
672
  if lazy:
667
673
  return self._to_polars_lazyframe(
668
674
  metadata=metadata,
@@ -762,6 +768,7 @@ class BaseFileReader(BaseFileIO, gc=False):
762
768
  pa.Table | list[pa.Table] | tuple[pa.Table | list[pa.Table], dict[str, Any]]: PyArrow Table or list of
763
769
  Tables and optional metadata.
764
770
  """
771
+ kwargs.pop("batch_size", None)
765
772
  self._load(
766
773
  reload=reload,
767
774
  metadata=metadata,
@@ -865,6 +872,7 @@ class BaseFileReader(BaseFileIO, gc=False):
865
872
  duckdb.DuckDBPyRelation | tuple[duckdb.DuckDBPyRelation, dict[str, Any]]: DuckDB relation and optional
866
873
  metadata.
867
874
  """
875
+ kwargs.pop("batch_size", None)
868
876
  if self._conn is None:
869
877
  if conn is None:
870
878
  conn = duckdb.connect()
@@ -924,6 +932,7 @@ class BaseFileReader(BaseFileIO, gc=False):
924
932
  duckdb.DuckDBPyConnection | tuple[duckdb.DuckDBPyConnection, dict[str, Any]]: DuckDB connection instance
925
933
  or DuckDB connection instance and optional metadata.
926
934
  """
935
+ kwargs.pop("batch_size", None)
927
936
  if name is None:
928
937
  name = f"{self.format}:{self.path}"
929
938
 
@@ -986,6 +995,7 @@ class BaseFileReader(BaseFileIO, gc=False):
986
995
  or DuckDB relation or connection instance and optional metadata.
987
996
 
988
997
  """
998
+ kwargs.pop("batch_size", None)
989
999
  if as_relation:
990
1000
  return self.to_duckdb_relation(
991
1001
  conn=conn,
@@ -1033,6 +1043,7 @@ class BaseFileReader(BaseFileIO, gc=False):
1033
1043
  Returns:
1034
1044
  None
1035
1045
  """
1046
+ kwargs.pop("batch_size", None)
1036
1047
  if name is None:
1037
1048
  name = f"{self.format}:{self.path}"
1038
1049
 
@@ -1048,7 +1059,9 @@ class BaseFileReader(BaseFileIO, gc=False):
1048
1059
  reload=reload,
1049
1060
  include_file_path=include_file_path,
1050
1061
  use_threads=use_threads,
1051
- opt_dtypes=opt_dtypes**kwargs,
1062
+ opt_dtypes=opt_dtypes,
1063
+ verbose=verbose,
1064
+ **kwargs,
1052
1065
  ).to_batches()
1053
1066
  ],
1054
1067
  )
@@ -141,6 +141,7 @@ def opt_dtype(
141
141
  exclude: str | list[str] | None = None,
142
142
  time_zone: str | None = None,
143
143
  shrink_numerics: bool = True,
144
+ strict: bool = False,
144
145
  ) -> pl.DataFrame:
145
146
  """
146
147
  Optimize data types of a Polars DataFrame for performance and memory efficiency.
@@ -155,6 +156,7 @@ def opt_dtype(
155
156
  exclude: Column(s) to exclude from optimization
156
157
  time_zone: Optional time zone for datetime parsing
157
158
  shrink_numerics: Whether to downcast numeric types when possible
159
+ strict: If True, will raise an error if any column cannot be optimized
158
160
 
159
161
  Returns:
160
162
  DataFrame with optimized data types
@@ -173,10 +175,17 @@ def opt_dtype(
173
175
  cols_to_process = [col for col in cols_to_process if col not in exclude]
174
176
 
175
177
  # Generate optimization expressions for all columns
176
- expressions = [
177
- _get_column_expr(df, col_name, shrink_numerics, time_zone)
178
- for col_name in cols_to_process
179
- ]
178
+ expressions = []
179
+ for col_name in cols_to_process:
180
+ try:
181
+ expressions.append(
182
+ _get_column_expr(df, col_name, shrink_numerics, time_zone)
183
+ )
184
+ except Exception as e:
185
+ if strict:
186
+ raise e
187
+ # If strict mode is off, just keep the original column
188
+ continue
180
189
 
181
190
  # Apply all transformations at once if any exist
182
191
  return df if not expressions else df.with_columns(expressions)
@@ -360,6 +360,7 @@ def opt_dtype(
360
360
  exclude: str | list[str] | None = None,
361
361
  time_zone: str | None = None,
362
362
  shrink_numerics: bool = True,
363
+ strict: bool = False,
363
364
  ) -> pa.Table:
364
365
  """
365
366
  Optimize data types of a PyArrow Table for performance and memory efficiency.
@@ -375,6 +376,7 @@ def opt_dtype(
375
376
  exclude: Column(s) to exclude from optimization
376
377
  time_zone: Optional time zone for datetime parsing
377
378
  shrink_numerics: Whether to downcast numeric types when possible
379
+ strict: If True, will raise an error if any column cannot be optimized
378
380
 
379
381
  Returns:
380
382
  PyArrow Table with optimized data types
@@ -396,9 +398,15 @@ def opt_dtype(
396
398
  new_columns = []
397
399
  for col_name in table.column_names:
398
400
  if col_name in cols_to_process:
399
- new_columns.append(
400
- _process_column(table, col_name, shrink_numerics, time_zone)
401
- )
401
+ try:
402
+ # Process column for optimization
403
+ new_columns.append(
404
+ _process_column(table, col_name, shrink_numerics, time_zone)
405
+ )
406
+ except Exception as e:
407
+ if strict:
408
+ raise e
409
+ new_columns.append(table[col_name])
402
410
  else:
403
411
  new_columns.append(table[col_name])
404
412
 
File without changes
File without changes
File without changes