FlowerPower 0.11.6.4__tar.gz → 0.11.6.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. {flowerpower-0.11.6.4/src/FlowerPower.egg-info → flowerpower-0.11.6.6}/PKG-INFO +1 -1
  2. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/pyproject.toml +1 -1
  3. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6/src/FlowerPower.egg-info}/PKG-INFO +1 -1
  4. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/fs/ext.py +12 -3
  5. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/helpers/polars.py +6 -6
  6. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/helpers/pyarrow.py +3 -3
  7. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/LICENSE +0 -0
  8. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/README.md +0 -0
  9. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/setup.cfg +0 -0
  10. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/FlowerPower.egg-info/SOURCES.txt +0 -0
  11. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/FlowerPower.egg-info/dependency_links.txt +0 -0
  12. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/FlowerPower.egg-info/entry_points.txt +0 -0
  13. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/FlowerPower.egg-info/requires.txt +0 -0
  14. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/FlowerPower.egg-info/top_level.txt +0 -0
  15. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/__init__.py +0 -0
  16. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/__init__.py +0 -0
  17. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/base.py +0 -0
  18. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/pipeline/__init__.py +0 -0
  19. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/pipeline/adapter.py +0 -0
  20. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/pipeline/run.py +0 -0
  21. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/pipeline/schedule.py +0 -0
  22. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/project/__init__.py +0 -0
  23. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/project/adapter.py +0 -0
  24. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cfg/project/job_queue.py +0 -0
  25. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cli/__init__.py +0 -0
  26. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cli/cfg.py +0 -0
  27. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cli/job_queue.py +0 -0
  28. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cli/mqtt.py +0 -0
  29. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cli/pipeline.py +0 -0
  30. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/cli/utils.py +0 -0
  31. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/flowerpower.py +0 -0
  32. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/fs/__init__.py +0 -0
  33. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/fs/base.py +0 -0
  34. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/fs/storage_options.py +0 -0
  35. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/__init__.py +0 -0
  36. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/apscheduler/__init__.py +0 -0
  37. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -0
  38. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -0
  39. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/apscheduler/manager.py +0 -0
  40. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/apscheduler/setup.py +0 -0
  41. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/apscheduler/trigger.py +0 -0
  42. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/apscheduler/utils.py +0 -0
  43. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/base.py +0 -0
  44. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/rq/__init__.py +0 -0
  45. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/rq/_trigger.py +0 -0
  46. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -0
  47. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -0
  48. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/rq/manager.py +0 -0
  49. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/rq/setup.py +0 -0
  50. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/job_queue/rq/utils.py +0 -0
  51. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/mqtt.py +0 -0
  52. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/pipeline/__init__.py +0 -0
  53. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/pipeline/base.py +0 -0
  54. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/pipeline/io.py +0 -0
  55. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/pipeline/job_queue.py +0 -0
  56. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/pipeline/manager.py +0 -0
  57. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/pipeline/registry.py +0 -0
  58. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/pipeline/runner.py +0 -0
  59. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/pipeline/visualizer.py +0 -0
  60. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/base.py +0 -0
  61. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/helpers/datetime.py +0 -0
  62. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/helpers/sql.py +0 -0
  63. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/__init__.py +0 -0
  64. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/csv.py +0 -0
  65. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/deltatable.py +0 -0
  66. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/duckdb.py +0 -0
  67. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/json.py +0 -0
  68. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/mqtt.py +0 -0
  69. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/mssql.py +0 -0
  70. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/mysql.py +0 -0
  71. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/oracle.py +0 -0
  72. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/parquet.py +0 -0
  73. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/postgres.py +0 -0
  74. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/pydala.py +0 -0
  75. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/loader/sqlite.py +0 -0
  76. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/metadata.py +0 -0
  77. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/__init__.py +0 -0
  78. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/csv.py +0 -0
  79. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/deltatable.py +0 -0
  80. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/duckdb.py +0 -0
  81. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/json.py +0 -0
  82. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/mqtt.py +0 -0
  83. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/mssql.py +0 -0
  84. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/mysql.py +0 -0
  85. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/oracle.py +0 -0
  86. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/parquet.py +0 -0
  87. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/postgres.py +0 -0
  88. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/pydala.py +0 -0
  89. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/io/saver/sqlite.py +0 -0
  90. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/mqtt/__init__.py +0 -0
  91. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/mqtt/cfg.py +0 -0
  92. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/plugins/mqtt/manager.py +0 -0
  93. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/settings/__init__.py +0 -0
  94. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/settings/backend.py +0 -0
  95. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/settings/executor.py +0 -0
  96. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/settings/general.py +0 -0
  97. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/settings/hamilton.py +0 -0
  98. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/settings/job_queue.py +0 -0
  99. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/settings/logging.py +0 -0
  100. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/settings/retry.py +0 -0
  101. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/utils/callback.py +0 -0
  102. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/utils/logging.py +0 -0
  103. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/utils/misc.py +0 -0
  104. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/utils/monkey.py +0 -0
  105. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/utils/open_telemetry.py +0 -0
  106. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/utils/scheduler.py +0 -0
  107. {flowerpower-0.11.6.4 → flowerpower-0.11.6.6}/src/flowerpower/utils/templates.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: FlowerPower
3
- Version: 0.11.6.4
3
+ Version: 0.11.6.6
4
4
  Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
5
5
  Author-email: "Volker L." <ligno.blades@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -4,7 +4,7 @@ description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower
4
4
  authors = [{ name = "Volker L.", email = "ligno.blades@gmail.com" }]
5
5
  readme = "README.md"
6
6
  requires-python = ">= 3.11"
7
- version = "0.11.6.4"
7
+ version = "0.11.6.6"
8
8
  keywords = [
9
9
  "hamilton",
10
10
  "workflow",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: FlowerPower
3
- Version: 0.11.6.4
3
+ Version: 0.11.6.6
4
4
  Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
5
5
  Author-email: "Volker L." <ligno.blades@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/legout/flowerpower
@@ -923,7 +923,10 @@ def _read_parquet(
923
923
  schemas = [t.schema for t in tables]
924
924
  unified_schema = unify_schemas_pa(schemas)
925
925
  tables = [cast_schema(t, unified_schema) for t in tables]
926
- result = pa.concat_tables(tables, promote_options="permissive")
926
+ result = pa.concat_tables(
927
+ [table for table in tables if table.num_rows > 0],
928
+ promote_options="permissive",
929
+ )
927
930
  # if opt_dtypes:
928
931
  # result = opt_dtype_pa(result, strict=False)
929
932
  return result
@@ -932,7 +935,10 @@ def _read_parquet(
932
935
  # tables = opt_dtype_pa(tables, strict=False)
933
936
  return tables
934
937
  else:
935
- return pa.concat_tables(tables, promote_options="permissive")
938
+ return pa.concat_tables(
939
+ [table for table in tables if table.num_rows > 0],
940
+ promote_options="permissive",
941
+ )
936
942
  return tables
937
943
 
938
944
 
@@ -1052,7 +1058,10 @@ def _read_parquet_batches(
1052
1058
  schemas = [t.schema for t in batch_tables]
1053
1059
  unified_schema = unify_schemas_pa(schemas)
1054
1060
  batch_tables = [cast_schema(t, unified_schema) for t in batch_tables]
1055
- result = pa.concat_tables(batch_tables, promote_options="permissive")
1061
+ result = pa.concat_tables(
1062
+ [table for table in batch_tables if table.num_rows > 0],
1063
+ promote_options="permissive",
1064
+ )
1056
1065
  # if opt_dtypes:
1057
1066
  # result = opt_dtype_pa(result, strict=False)
1058
1067
  yield result
@@ -68,20 +68,20 @@ def _optimize_string_column(
68
68
  cleaned_expr = _clean_string_expr(col_name)
69
69
  non_null = series.drop_nulls().replace({"-": None, "": None, "None": None})
70
70
  if len(non_null) == 0:
71
- return pl.col(col_name).cast(pl.Int8)
71
+ return pl.col(col_name).cast(series.dtype)
72
72
 
73
73
  stripped = non_null.str.strip_chars()
74
74
  lowercase = stripped.str.to_lowercase()
75
75
 
76
76
  # Check for boolean values
77
- if lowercase.str.contains(BOOLEAN_REGEX).all():
77
+ if lowercase.str.contains(BOOLEAN_REGEX).all(ignore_nulls=False):
78
78
  return (
79
79
  cleaned_expr.str.to_lowercase()
80
80
  .str.contains(BOOLEAN_TRUE_REGEX)
81
81
  .alias(col_name)
82
82
  )
83
83
 
84
- elif stripped.str.contains(INTEGER_REGEX).all():
84
+ elif stripped.str.contains(INTEGER_REGEX).all(ignore_nulls=False):
85
85
  int_expr = cleaned_expr.cast(pl.Int64)
86
86
  return (
87
87
  int_expr.shrink_dtype().alias(col_name)
@@ -90,7 +90,7 @@ def _optimize_string_column(
90
90
  )
91
91
 
92
92
  # Check for numeric values
93
- elif stripped.str.contains(FLOAT_REGEX).all():
93
+ elif stripped.str.contains(FLOAT_REGEX).all(ignore_nulls=False):
94
94
  float_expr = cleaned_expr.str.replace_all(",", ".").cast(pl.Float64)
95
95
 
96
96
  if shrink_numerics:
@@ -104,7 +104,7 @@ def _optimize_string_column(
104
104
  return float_expr.alias(col_name)
105
105
 
106
106
  try:
107
- if stripped.str.contains(DATETIME_REGEX).all():
107
+ if stripped.str.contains(DATETIME_REGEX).all(ignore_nulls=False):
108
108
  return cleaned_expr.str.to_datetime(
109
109
  strict=False, time_unit="us", time_zone=time_zone
110
110
  ).alias(col_name)
@@ -123,7 +123,7 @@ def _get_column_expr(
123
123
 
124
124
  # Handle all-null columns
125
125
  if series.is_null().all():
126
- return pl.col(col_name).cast(pl.Int8)
126
+ return pl.col(col_name).cast(series.dtype)
127
127
 
128
128
  # Process based on current type
129
129
  if series.dtype.is_numeric():
@@ -255,7 +255,7 @@ def _all_match_regex(array: pa.Array, pattern: str) -> bool:
255
255
  Uses pyarrow.compute.match_substring_regex for vectorized evaluation.
256
256
  """
257
257
  if len(array) == 0 or array.null_count == len(array):
258
- return True
258
+ return False
259
259
 
260
260
  # Check if al values match the pattern
261
261
  return pc.all(pc.match_substring_regex(array, pattern, ignore_case=True)).as_py()
@@ -272,7 +272,7 @@ def _optimize_string_array(
272
272
  if len(array) == 0:
273
273
  return pa.array([], type=pa.int8())
274
274
  if array.null_count == len(array):
275
- return pa.array([None] * len(array), type=pa.int8())
275
+ return pa.array([None] * len(array), type=array.type)
276
276
 
277
277
  # Clean string values
278
278
  cleaned_array = _clean_string_array(array)
@@ -342,7 +342,7 @@ def _process_column(
342
342
 
343
343
  # Handle all-null columns
344
344
  if array.null_count == len(array):
345
- return pa.array([None] * len(array), type=pa.int8())
345
+ return pa.array([None] * len(array), type=array.type)
346
346
 
347
347
  # Process based on current type
348
348
  if pa.types.is_floating(array.type) or pa.types.is_integer(array.type):
File without changes
File without changes
File without changes