fpstreams 2.4.0__tar.gz → 2.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fpstreams-2.4.0 → fpstreams-2.5.0}/CHANGELOG.md +222 -2
- {fpstreams-2.4.0 → fpstreams-2.5.0}/PKG-INFO +123 -54
- {fpstreams-2.4.0 → fpstreams-2.5.0}/README.md +117 -53
- {fpstreams-2.4.0 → fpstreams-2.5.0}/pyproject.toml +28 -5
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/Cargo.lock +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/Cargo.toml +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/common.rs +12 -2
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/float/endpoints.rs +34 -35
- fpstreams-2.5.0/rust/src/float/rolling.rs +155 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/float.rs +3 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/integer/endpoints.rs +92 -15
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/integer.rs +2 -2
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/lib.rs +8 -7
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/numeric/buffers.rs +47 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/numeric/reductions.rs +70 -2
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/__init__.py +12 -12
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/_native.pyi +6 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/aggregate.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/async_flow.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/collecting/_collector_base.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/collecting/aggregate_program.py +1 -6
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/collecting/aggregation.py +24 -7
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/collecting/collector.py +533 -47
- fpstreams-2.5.0/src/fpstreams/collecting/sketches.py +122 -0
- fpstreams-2.5.0/src/fpstreams/collecting/statistics.py +383 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/collectors.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/column.py +1 -1
- fpstreams-2.5.0/src/fpstreams/errors.py +72 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/exceptions.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/_rows_fusion.py +3 -7
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/_scalar_fusion.py +4 -15
- fpstreams-2.4.0/src/fpstreams/execution/arrow.py → fpstreams-2.5.0/src/fpstreams/execution/arrow/__init__.py +107 -371
- fpstreams-2.5.0/src/fpstreams/execution/arrow/numeric.py +335 -0
- fpstreams-2.5.0/src/fpstreams/execution/arrow/sort.py +129 -0
- fpstreams-2.4.0/src/fpstreams/execution/async_scheduler.py → fpstreams-2.5.0/src/fpstreams/execution/asynchronous/__init__.py +85 -17
- fpstreams-2.5.0/src/fpstreams/execution/asynchronous/concurrent.py +288 -0
- fpstreams-2.4.0/src/fpstreams/execution/async_merge.py → fpstreams-2.5.0/src/fpstreams/execution/asynchronous/merge.py +16 -8
- fpstreams-2.4.0/src/fpstreams/execution/async_iterators.py → fpstreams-2.5.0/src/fpstreams/execution/asynchronous/operators.py +119 -14
- fpstreams-2.4.0/src/fpstreams/execution/async_timers.py → fpstreams-2.5.0/src/fpstreams/execution/asynchronous/timers.py +9 -9
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/native.py +2 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/numpy_group.py +6 -18
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/numpy_prefix.py +1 -4
- fpstreams-2.4.0/src/fpstreams/execution/_pair_aggregate.py → fpstreams-2.5.0/src/fpstreams/execution/pairs/__init__.py +2 -2
- fpstreams-2.4.0/src/fpstreams/execution/_pair_dict.py → fpstreams-2.5.0/src/fpstreams/execution/pairs/dictionary.py +43 -32
- fpstreams-2.4.0/src/fpstreams/execution/_pair_row_filter.py → fpstreams-2.5.0/src/fpstreams/execution/pairs/row_filter.py +13 -17
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/physical.py +7 -7
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/relational/__init__.py +32 -562
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/relational/arrow_global.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/relational/arrow_group.py +3 -2
- fpstreams-2.5.0/src/fpstreams/execution/relational/collector_group.py +202 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/relational/join.py +8 -7
- fpstreams-2.5.0/src/fpstreams/execution/relational/native_group.py +423 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/sync.py +10 -3
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/sync_ops.py +81 -26
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expr.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/_codegen.py +20 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/_row_codegen.py +6 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/program.py +29 -26
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/row.py +19 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/row_eval.py +17 -5
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/selectors.py +67 -20
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/typed_ir.py +5 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/gather.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/io_safety.py +12 -3
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/option.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/physical/compiled.py +1 -3
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/physical/kernel_cache.py +3 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/physical/plan.py +2 -0
- fpstreams-2.4.0/src/fpstreams/planning/arrow.py → fpstreams-2.5.0/src/fpstreams/planning/arrow/__init__.py +24 -12
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/async_.py +109 -11
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/async_utils.py +3 -1
- fpstreams-2.5.0/src/fpstreams/planning/compiler.py +422 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/explain.py +21 -6
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/gather.py +5 -1
- fpstreams-2.5.0/src/fpstreams/planning/lineage.py +308 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/logical.py +20 -2
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/native.py +7 -7
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/numpy.py +1 -1
- fpstreams-2.4.0/src/fpstreams/planning/pair_i64_expression.py → fpstreams-2.5.0/src/fpstreams/planning/pairs/i64_expression.py +32 -8
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/plan_cache.py +2 -0
- fpstreams-2.5.0/src/fpstreams/planning/relational/__init__.py +316 -0
- fpstreams-2.4.0/src/fpstreams/planning/compiler.py → fpstreams-2.5.0/src/fpstreams/planning/relational/specs.py +16 -684
- fpstreams-2.4.0/src/fpstreams/planning/semantics.py → fpstreams-2.5.0/src/fpstreams/planning/semantics/__init__.py +6 -1
- fpstreams-2.4.0/src/fpstreams/planning/semantic_rules.py → fpstreams-2.5.0/src/fpstreams/planning/semantics/analysis.py +301 -33
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/source.py +3 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/sync.py +13 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/result.py +1 -1
- fpstreams-2.5.0/src/fpstreams/runtime/_distinct.py +124 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/iterators.py +58 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/query.py +8 -2
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/resources.py +30 -26
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/tasks.py +49 -17
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/storage/checkpoint.py +34 -14
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/storage/spill_store.py +3 -2
- fpstreams-2.5.0/src/fpstreams/streams/_terminals/__init__.py +423 -0
- fpstreams-2.5.0/src/fpstreams/streams/_terminals/reductions.py +528 -0
- fpstreams-2.4.0/src/fpstreams/streams/_flow_structural_list.py → fpstreams-2.5.0/src/fpstreams/streams/_terminals/structural.py +48 -45
- fpstreams-2.4.0/src/fpstreams/streams/_flow_unique_list.py → fpstreams-2.5.0/src/fpstreams/streams/_terminals/unique.py +36 -19
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/streams/async_flow.py +149 -25
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/streams/async_terminals.py +69 -20
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/streams/flow.py +65 -21
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/streams/flow_terminals.py +237 -841
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/streams/pairs.py +46 -19
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/arrow.py +2 -6
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/dataframe.py +10 -6
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/factory.py +5 -7
- fpstreams-2.4.0/src/fpstreams/tabular/io.py → fpstreams-2.5.0/src/fpstreams/tabular/io/__init__.py +38 -55
- fpstreams-2.5.0/src/fpstreams/tabular/io/database.py +559 -0
- fpstreams-2.4.0/src/fpstreams/tabular/_text_sources.py → fpstreams-2.5.0/src/fpstreams/tabular/io/text.py +25 -12
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/join.py +8 -11
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/numpy.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/polars.py +8 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/rows.py +82 -82
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/spill.py +6 -21
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/spill_io.py +1 -1
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/testing.py +19 -6
- fpstreams-2.4.0/src/fpstreams/collecting/statistics.py +0 -194
- fpstreams-2.4.0/src/fpstreams/errors.py +0 -33
- fpstreams-2.4.0/src/fpstreams/execution/async_map.py +0 -92
- fpstreams-2.4.0/src/fpstreams/execution/async_ops.py +0 -97
- fpstreams-2.4.0/src/fpstreams/execution/async_prefetch.py +0 -105
- fpstreams-2.4.0/src/fpstreams/execution/async_queue.py +0 -94
- fpstreams-2.4.0/src/fpstreams/planning/semantic_analyzer.py +0 -168
- fpstreams-2.4.0/src/fpstreams/runtime/_distinct.py +0 -35
- fpstreams-2.4.0/src/fpstreams/tabular/sql.py +0 -236
- fpstreams-2.4.0/src/fpstreams/tabular/sqlite_sink.py +0 -381
- {fpstreams-2.4.0 → fpstreams-2.5.0}/LICENSE +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/build.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/float/affine_pair.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/numeric_mean.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/numpy_export.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/numpy_group/buffer.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/numpy_group.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/pair/expr.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/pair/prefix.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/pair/row_filter.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/pair/unique.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/pair/value_filter.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/pair/value_map.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/pair.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/pivot.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/records.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/adapters/namedtuple.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/adapters.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/global_numeric.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/group_numeric.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/group_pair_expr.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/join_callable/many.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/join_callable/unique.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/join_callable.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/join_exact.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational/join_i64.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational_fixed/composite.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational_fixed/global_multi/same_field.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational_fixed/global_multi.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational_fixed/group_multi.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational_fixed/single.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/relational_fixed.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/scalar_sort.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/scalar_unique.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/select.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/adapters.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/group_exact/pairs.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/group_exact/rows.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/group_exact.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/group_fixed.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/join_callable/behavior.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/join_callable/direct.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/join_callable.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/join_exact.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/numeric/kernels.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/numeric.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests/numpy_group.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/tests.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/unnest.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/rust/src/unpivot.rs +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/_provenance.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/collecting/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/collecting/program.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/collecting/reducer.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/relational/arrow_group_rows.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/sorted_streams.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/sorting.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/execution/window_reduce.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/row_ir.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/expressions/scalar.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/functional.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/physical/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/physical/async_plan.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/physical/relational.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/planning/__init__.py +0 -0
- /fpstreams-2.4.0/src/fpstreams/planning/arrow_source.py → /fpstreams-2.5.0/src/fpstreams/planning/arrow/source.py +0 -0
- /fpstreams-2.4.0/src/fpstreams/planning/_pair_stages.py → /fpstreams-2.5.0/src/fpstreams/planning/pairs/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/primitives/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/primitives/option.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/primitives/result.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/py.typed +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/executor.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/failpoints.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/files.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/limits.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/metrics.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/report.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/runtime/spill.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/storage/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/storage/codec.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/streams/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/streams/_async_identity.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/__init__.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/grouped.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/records.py +0 -0
- {fpstreams-2.4.0 → fpstreams-2.5.0}/src/fpstreams/tabular/spill_limits.py +0 -0
|
@@ -1,9 +1,229 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
This file records user-visible and compatibility-relevant changes in fpstreams
|
|
3
|
+
This file records user-visible and compatibility-relevant changes in fpstreams,
|
|
4
4
|
including changed defaults.
|
|
5
5
|
|
|
6
|
-
##
|
|
6
|
+
## 2.5.0
|
|
7
|
+
|
|
8
|
+
Version 2.5.0 includes an intentional static typing incompatibility: `Collectors.averaging()` now
|
|
9
|
+
returns `object` in type annotations rather than incorrectly promising `float`.
|
|
10
|
+
Runtime arithmetic is unchanged. Use `Collectors.averaging_with()` for explicit
|
|
11
|
+
accumulator and result types, or narrow the legacy result before use.
|
|
12
|
+
|
|
13
|
+
Dynamic `reduce()` remains dynamically typed. For an explicit accumulator
|
|
14
|
+
contract, use an annotated `Flow.fold()` result or `AsyncFlow.fold_async()`.
|
|
15
|
+
The migration examples are in the collecting and async flow reference pages;
|
|
16
|
+
2.5.0 does not claim to solve automatic `reduce()` inference.
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- `AsyncFlow.fold_async()` provides an explicit asynchronous state contract.
|
|
21
|
+
Use annotated `Flow.fold()` or `AsyncFlow.fold_async()` results when migrating
|
|
22
|
+
dynamic `reduce()` calls to a declared accumulator type.
|
|
23
|
+
- `Collectors.averaging_with()` accepts typed accumulation, division, and empty
|
|
24
|
+
result callbacks for numeric types and custom arithmetic.
|
|
25
|
+
- `Rows.drop_nulls(missing="raise")` propagates lookup failures. Callable
|
|
26
|
+
selectors can define actual absence by returning `None`; the default
|
|
27
|
+
`missing="legacy"` preserves historical lookup handling.
|
|
28
|
+
- `Collectors.averaging()` now declares its result as `object`, replacing the
|
|
29
|
+
inaccurate `float` annotation. Runtime arithmetic is unchanged; migrate typed
|
|
30
|
+
consumers to `averaging_with()` or narrow the result explicitly.
|
|
31
|
+
- `Flow.chunk()` and `AsyncFlow.chunk()` accept `strict=True` to reject an
|
|
32
|
+
incomplete final batch during consumption. The `batch` aliases also support it.
|
|
33
|
+
- `Collectors.frequencies()` counts hashable inputs in first-encounter order and
|
|
34
|
+
composes with synchronous, asynchronous, and grouped collectors.
|
|
35
|
+
- `Collectors.mode()` finds the most frequent hashable value in sync, async,
|
|
36
|
+
and grouped collection, with first-encountered tie breaking.
|
|
37
|
+
- Extend experimental free-threaded core-concurrency CI to Python 3.15t alongside
|
|
38
|
+
3.14t. Check that native import keeps the GIL disabled, and keep optional adapter
|
|
39
|
+
dependencies out of these jobs through the shared `test-core` group.
|
|
40
|
+
- Add standard CPython 3.15 to the runtime and consumer typing CI matrices.
|
|
41
|
+
The frozen data-adapter environment now uses pandas 3.0.6 and PyArrow 26.0.0
|
|
42
|
+
with Python 3.15 wheels.
|
|
43
|
+
- Runnable resource-ownership examples cover early termination, callback errors,
|
|
44
|
+
and caller cancellation. CI executes their documented code without optional extras.
|
|
45
|
+
- `Collectors.statistics(selector, ddof=0)` computes count, mean, variance, and
|
|
46
|
+
standard deviation with one selection and conversion per valid item.
|
|
47
|
+
- Optional `fpstreams[sketch]` collectors provide KLL approximate quantiles and
|
|
48
|
+
HLL approximate distinct counts through the existing sync, async, and grouped
|
|
49
|
+
reduction interfaces. Exact collectors keep their existing behavior.
|
|
50
|
+
- `Flow.rolling_statistics(size, selector=..., ddof=0)` and its async equivalent
|
|
51
|
+
emit count, mean, and variance for recent items, including warmup windows.
|
|
52
|
+
They reuse `map_accum`, with independent state for every evaluation.
|
|
53
|
+
- `Rows.explain().to_dict(include_field_dependencies=True)` exports static field dependencies.
|
|
54
|
+
Direct projections, computed columns, renames, and supported relational
|
|
55
|
+
operations retain source identities; unsupported operations report unknown
|
|
56
|
+
dependencies. Value inputs and row selection inputs are reported separately.
|
|
57
|
+
|
|
58
|
+
### Performance
|
|
59
|
+
|
|
60
|
+
- Use C-backed batching for eligible strict retained-list chunks and exact
|
|
61
|
+
generator, map, and filter cursors on Python 3.12+. Python 3.11 and custom
|
|
62
|
+
cursors retain their iterator path.
|
|
63
|
+
- Reduce intermediate generator calls in legacy Gatherers and selected-field
|
|
64
|
+
`Rows.drop_nulls()` while preserving lookup errors and short-circuiting.
|
|
65
|
+
- Use an optional Rust numeric window state for rolling statistics. Selection
|
|
66
|
+
and numeric conversion remain in Python. Explicit Python execution and
|
|
67
|
+
extensions without this capability retain the reference implementation.
|
|
68
|
+
|
|
69
|
+
### Fixed
|
|
70
|
+
|
|
71
|
+
- Preserve the original custom-key hash exception in synchronous and asynchronous
|
|
72
|
+
per-key reductions, frequency counts, Pair collection, and in-memory Python
|
|
73
|
+
joins and grouping on CPython 3.14 and 3.15. Keep hash calls, raw key identity, and source cleanup unchanged.
|
|
74
|
+
|
|
75
|
+
- Reuse each compiled scalar stage's validated fingerprint for the physical-plan
|
|
76
|
+
cache instead of encoding the same expression again. Row expressions remain
|
|
77
|
+
outside structural caches.
|
|
78
|
+
|
|
79
|
+
- NumPy filter planning checks numeric classes by identity, preserving Python
|
|
80
|
+
fallback for custom integer types whose metaclasses define hashing or equality.
|
|
81
|
+
- Materializing deep `RowExpr` maps and filters skips the unused structural
|
|
82
|
+
fingerprint and reaches the iterative evaluator when direct compilation is bounded.
|
|
83
|
+
- Row-expression effect analysis visits shared IR nodes once. Evaluation still
|
|
84
|
+
executes each occurrence, including repeated selectors and callbacks.
|
|
85
|
+
|
|
86
|
+
- Close the iterator returned by custom Rows/Flow iteration in `to_columns()`
|
|
87
|
+
after exhaustion or conversion failure, preserving the primary error.
|
|
88
|
+
- Respect custom row iteration in `arrow_batches()` with default or explicit
|
|
89
|
+
schemas. Check iteration at consumption time while retaining eligible native
|
|
90
|
+
paths and bounded batch conversion.
|
|
91
|
+
|
|
92
|
+
- Finish a cancelled child's cleanup before propagating cancellation of its drain
|
|
93
|
+
waiter. Repeated caller cancellation no longer interrupts that cleanup or lets
|
|
94
|
+
timeout operators close the source before its pending pull has stopped.
|
|
95
|
+
|
|
96
|
+
- Preserve each result when a shared row-expression node is evaluated more than
|
|
97
|
+
once. Stateful callbacks and dynamic selectors now agree with direct evaluation
|
|
98
|
+
in the iterative fallback, including nested lazy operands.
|
|
99
|
+
|
|
100
|
+
- Convert Arrow extension columns on pandas 2.0 without the dataframe interchange
|
|
101
|
+
protocol when copying is allowed. Keep conversion lazy and exclude index columns.
|
|
102
|
+
- Preserve zero-column row counts in Polars output, including bounded batches,
|
|
103
|
+
on the declared minimum Polars version.
|
|
104
|
+
|
|
105
|
+
- Keep generated row and scalar functions compatible with Python 3.11 AST types.
|
|
106
|
+
|
|
107
|
+
- Reject NaN async durations before opening a source or starting work. Zero stays
|
|
108
|
+
valid for `interval` and `debounce`. An invalid timer rearm preserves its
|
|
109
|
+
existing timer task.
|
|
110
|
+
|
|
111
|
+
- Normalize integer counts for `merge_map` concurrency and synchronous parallel
|
|
112
|
+
workers/buffers before execution. Noninteger values raise `TypeError`, and later
|
|
113
|
+
mutation of an integer-like configuration object cannot change the stored limit.
|
|
114
|
+
|
|
115
|
+
- Require integer capacities for plan and kernel caches, normalizing integer-like
|
|
116
|
+
values once so later mutation cannot change the configured bound.
|
|
117
|
+
|
|
118
|
+
- Accept inner async iterators without an `aclose` hook in `merge_map` and
|
|
119
|
+
`switch_map`. Optional hooks may return an awaitable or a synchronous value;
|
|
120
|
+
cleanup retains the hook selected when the inner opens.
|
|
121
|
+
|
|
122
|
+
- Close the owned outer source when `switch_map` cannot submit its first pull
|
|
123
|
+
task, retaining the submission error and any source cleanup diagnostics.
|
|
124
|
+
Root ownership also covers unstarted intermediate stages and sync-to-async
|
|
125
|
+
adapters without consuming an item or repeating source close.
|
|
126
|
+
|
|
127
|
+
- Drain existing scope tasks before closing upstream resources when a task factory
|
|
128
|
+
rejects the cleanup task. The original submission error remains observable.
|
|
129
|
+
|
|
130
|
+
- Preserve JSON Lines record boundaries when disabling the byte limit. Managed
|
|
131
|
+
paths now use LF/CRLF framing in both modes; borrowed text handles retain the
|
|
132
|
+
caller's newline handling.
|
|
133
|
+
|
|
134
|
+
- Require an initial BOM for nonempty JSON Lines input using generic UTF-16/32
|
|
135
|
+
codecs, consistently across byte-limit modes. Explicit byte-order codecs
|
|
136
|
+
continue to support data without a BOM.
|
|
137
|
+
|
|
138
|
+
- Handle UTF-8 signatures consistently across bounded and unbounded JSON Lines
|
|
139
|
+
sources. A signature before a later record is no longer silently discarded.
|
|
140
|
+
|
|
141
|
+
- Wait for in-flight asynchronous file operations before completing cancellation
|
|
142
|
+
or closing their handles, including cancellation during file acquisition.
|
|
143
|
+
|
|
144
|
+
- Preserve asynchronous file read errors and caller cancellation when file close
|
|
145
|
+
also fails. Close failures still propagate after exhaustion or early termination.
|
|
146
|
+
|
|
147
|
+
- Preserve serialization errors when text writer and source cleanup also fail.
|
|
148
|
+
Atomic CSV, JSON, and JSON Lines exports keep the previous destination on failure.
|
|
149
|
+
|
|
150
|
+
- Preserve checkpoint callback failures when staging cleanup or lock-file close also
|
|
151
|
+
fails. Retries verify and reuse entries published before a later cleanup error.
|
|
152
|
+
|
|
153
|
+
- Clarify how asynchronous `reduce`, `reduce_right`, and `fold_right` resolve
|
|
154
|
+
reducer results while preserving initial values and singleton items unchanged.
|
|
155
|
+
|
|
156
|
+
- Preserve operation errors and continue closing owned resources when a cleanup
|
|
157
|
+
exception raises while formatting its message.
|
|
158
|
+
|
|
159
|
+
- Preserve element and fallback types for synchronous and asynchronous `first`,
|
|
160
|
+
`last`, `find`, and `nth` results. Calls without a default retain the element type.
|
|
161
|
+
|
|
162
|
+
- Retain annotated container Collector results for `Any` input elements and
|
|
163
|
+
infer awaited callable results in async collection. Optional-collector
|
|
164
|
+
forwarding and named results keep their existing contracts.
|
|
165
|
+
- Preserve inferred Collector element/result types for direct container factories
|
|
166
|
+
while retaining explicit `Collectors[int]` construction. Per-key collection
|
|
167
|
+
keeps the collector state type; input variance reflects consumption.
|
|
168
|
+
- Reject non-real numeric scalars such as NumPy `complex64` before converting
|
|
169
|
+
statistical inputs or quantile ranks to floats. Decimal, Fraction, and real
|
|
170
|
+
NumPy scalar inputs keep their existing support.
|
|
171
|
+
- Concurrent async mapping can emit completed results while the source is still
|
|
172
|
+
waiting to produce another item. With concurrency above one, a reader task
|
|
173
|
+
also counts against the query's task limit.
|
|
174
|
+
- `map_async()` normalizes `buffer` through the integer-index protocol when the
|
|
175
|
+
pipeline is built, rejecting noninteger bounds before opening the source.
|
|
176
|
+
- Python and Rust variance calculations retain small differences in data with a
|
|
177
|
+
large common offset. Existing nonfinite-input policies remain unchanged.
|
|
178
|
+
- `Flow.to_async()` preserves its item type, and `Flow.pairs()` preserves tuple
|
|
179
|
+
key and value types. Dynamic pair inputs keep their existing runtime support.
|
|
180
|
+
- Native integer and floating-point range endpoints periodically check Python
|
|
181
|
+
signals during long scans.
|
|
182
|
+
- Reuse native range bounds for periodic signal checks, avoiding a separate
|
|
183
|
+
per-item countdown while preserving the 4096-input check interval.
|
|
184
|
+
|
|
185
|
+
### Changed
|
|
186
|
+
|
|
187
|
+
- Retained primitive Arrow sources with a complete compatible columnar prefix
|
|
188
|
+
collect directly into `to_list()`, preserving conversion and cleanup hooks.
|
|
189
|
+
- Optional field-dependency export reuses each computed-column analysis within
|
|
190
|
+
its step while preserving opaque-expression short circuiting.
|
|
191
|
+
|
|
192
|
+
- Keep performance runners and their regression fixtures local to maintainers.
|
|
193
|
+
Public source distributions and CI retain library checks without benchmark
|
|
194
|
+
components or hosted timing jobs.
|
|
195
|
+
- Consolidate internal async execution, pair execution, semantic analysis,
|
|
196
|
+
relational planning, terminal helpers, Arrow planning/execution, pair planning,
|
|
197
|
+
and I/O into domain packages. Public
|
|
198
|
+
imports remain unchanged; private implementation paths have moved.
|
|
199
|
+
- Release excess native numeric vector capacity before constructing list, tuple,
|
|
200
|
+
or set results, reducing overlapping temporary storage.
|
|
201
|
+
- Read row-stage metadata without a shared registry lock. Registration and
|
|
202
|
+
reclamation remain synchronized, with weak owner and executable-code checks.
|
|
203
|
+
- Reuse expression analysis within each field-dependency export. Spill partition
|
|
204
|
+
writes consume their existing byte buffers without an intermediate copy.
|
|
205
|
+
- Guarded Arrow projections can retain builtin scalar and list columns after
|
|
206
|
+
full buffer validation. Unsupported nested types retain row conversion.
|
|
207
|
+
- Plan reasons distinguish representation selection from terminal execution.
|
|
208
|
+
- Separate Arrow numeric reductions and retained sorting from batch dispatch,
|
|
209
|
+
preserving the existing execution entry points and live callback bindings.
|
|
210
|
+
- Document the existing v2 `compact()` difference between Flow and AsyncFlow;
|
|
211
|
+
use `filter_none()` to retain falsey values other than `None` in both domains.
|
|
212
|
+
- Correct CSV scan documentation: query projection follows conversion of all
|
|
213
|
+
caller-visible columns. Explicit Arrow conversion options can restrict the
|
|
214
|
+
reader's columns.
|
|
215
|
+
- Release qualification reuses CI before building distributions. New Python
|
|
216
|
+
3.11 adapter jobs exercise `lowest-direct` and `highest` dependency resolutions;
|
|
217
|
+
Windows and macOS smoke selections include async cancellation and executor
|
|
218
|
+
ownership paths. Hosted qualification results remain to be established.
|
|
219
|
+
|
|
220
|
+
## 2.4.1 - 2026-10-18
|
|
221
|
+
|
|
222
|
+
### Changed
|
|
223
|
+
|
|
224
|
+
- Separate relational candidate recognition and tree compilation from the general
|
|
225
|
+
compiler. Native grouping, Collector grouping, and synchronous terminal helpers
|
|
226
|
+
now live in domain modules; public interfaces and execution rules stay the same.
|
|
7
227
|
|
|
8
228
|
## 2.4.0 - 2026-10-18
|
|
9
229
|
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fpstreams
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Programming Language :: Python :: 3
|
|
6
6
|
Classifier: Programming Language :: Python :: 3.11
|
|
7
7
|
Classifier: Programming Language :: Python :: 3.12
|
|
8
8
|
Classifier: Programming Language :: Python :: 3.13
|
|
9
9
|
Classifier: Programming Language :: Python :: 3.14
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
10
11
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
11
12
|
Classifier: License :: OSI Approved :: MIT License
|
|
12
13
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
13
14
|
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: typing-extensions>=4.12 ; python_full_version < '3.13'
|
|
16
|
+
Requires-Dist: datasketches>=5.2,<6 ; extra == 'all'
|
|
14
17
|
Requires-Dist: hypothesis>=6.138,<7 ; extra == 'all'
|
|
15
18
|
Requires-Dist: pytest>=9.0.3,<10 ; extra == 'all'
|
|
16
19
|
Requires-Dist: pytest-cov>=7,<8 ; extra == 'all'
|
|
@@ -27,6 +30,7 @@ Requires-Dist: pandas>=2 ; extra == 'data'
|
|
|
27
30
|
Requires-Dist: pyarrow>=16 ; extra == 'data'
|
|
28
31
|
Requires-Dist: polars>=1.34 ; extra == 'polars'
|
|
29
32
|
Requires-Dist: pyarrow>=16 ; extra == 'polars'
|
|
33
|
+
Requires-Dist: datasketches>=5.2,<6 ; extra == 'sketch'
|
|
30
34
|
Requires-Dist: hypothesis>=6.138,<7 ; extra == 'test'
|
|
31
35
|
Requires-Dist: pytest>=9.0.3,<10 ; extra == 'test'
|
|
32
36
|
Requires-Dist: pytest-cov>=7,<8 ; extra == 'test'
|
|
@@ -40,6 +44,7 @@ Provides-Extra: arrow
|
|
|
40
44
|
Provides-Extra: async
|
|
41
45
|
Provides-Extra: data
|
|
42
46
|
Provides-Extra: polars
|
|
47
|
+
Provides-Extra: sketch
|
|
43
48
|
Provides-Extra: test
|
|
44
49
|
License-File: LICENSE
|
|
45
50
|
Summary: Lazy Python pipelines for iterables and records, with bounded async concurrency.
|
|
@@ -53,21 +58,34 @@ Project-URL: Changelog, https://github.com/steventimes/fpstreams/blob/master/CHA
|
|
|
53
58
|
Project-URL: Documentation, https://steventimes.github.io/fpstreams/
|
|
54
59
|
Project-URL: Homepage, https://github.com/steventimes/fpstreams
|
|
55
60
|
|
|
61
|
+

|
|
62
|
+
|
|
56
63
|
# fpstreams
|
|
57
64
|
|
|
65
|
+
[Install](#install) · [Quick start](#quick-start) · [Examples](#what-is-in-v2) · [Engines](#execution-engines) · [Documentation](https://steventimes.github.io/fpstreams/)
|
|
66
|
+
|
|
67
|
+
| Work with | Start with | Execution |
|
|
68
|
+
| --- | --- | --- |
|
|
69
|
+
| Values and records | `flow()` | Lazy synchronous pipelines |
|
|
70
|
+
| Async inputs and calls | `aflow()` | Bounded concurrent work |
|
|
71
|
+
| Key/value data | `pairs()` | Per-key collection and aggregation |
|
|
72
|
+
|
|
58
73
|
[](https://github.com/steventimes/fpstreams/actions/workflows/ci.yml)
|
|
59
74
|
[](https://pypi.org/project/fpstreams/)
|
|
60
75
|
[](https://pypi.org/project/fpstreams/)
|
|
61
76
|
[](https://github.com/steventimes/fpstreams/blob/master/LICENSE)
|
|
77
|
+
[](https://hub.xerj.org/contributors/steventimes)
|
|
62
78
|
|
|
63
79
|
[Documentation](https://steventimes.github.io/fpstreams/) · [Browser playground](https://steventimes.github.io/fpstreams/playground/) · [Changelog](https://github.com/steventimes/fpstreams/blob/master/CHANGELOG.md) · [Contributing](https://github.com/steventimes/fpstreams/blob/master/CONTRIBUTING.md)
|
|
64
80
|
|
|
65
|
-
|
|
81
|
+
Process Python objects and async inputs with lazy pipelines, bounded concurrent
|
|
82
|
+
work, and explicit resource cleanup.
|
|
66
83
|
|
|
67
84
|
Python 3.11 or newer is required.
|
|
68
85
|
|
|
69
|
-
|
|
70
|
-
|
|
86
|
+
[Version 2.4.1](https://github.com/steventimes/fpstreams/releases/tag/v2.4.1)
|
|
87
|
+
reorganizes planner and execution internals with no public API changes.
|
|
88
|
+
Version 2.4.0 added stateful pipeline operations, composed aggregation, shared execution
|
|
71
89
|
limits, file recovery, and versioned execution metadata. See the
|
|
72
90
|
[changelog](https://github.com/steventimes/fpstreams/blob/master/CHANGELOG.md) for its scope and the
|
|
73
91
|
[performance guide](https://steventimes.github.io/fpstreams/user-guide/performance/#process-mapping-and-explicit-batches)
|
|
@@ -75,9 +93,21 @@ for batch and executor examples. The
|
|
|
75
93
|
[roadmap](https://steventimes.github.io/fpstreams/roadmap/#next-development-steps)
|
|
76
94
|
lists the next optimization candidates and their acceptance criteria.
|
|
77
95
|
|
|
96
|
+
Version 2.5.0 preserves custom-key hash errors in per-key reductions, Pair
|
|
97
|
+
collection, and in-memory Python joins and grouping on Python 3.14 and 3.15.
|
|
98
|
+
|
|
99
|
+
The [2.5.0 changes](CHANGELOG.md#250) include a static averaging type correction
|
|
100
|
+
with an explicit migration path. This version adds
|
|
101
|
+
[typed asynchronous folds](fpstreams/docs/reference/async_flow.md#explicit-asynchronous-accumulator-contract-250),
|
|
102
|
+
[explicit averaging operations](fpstreams/docs/reference/collecting.md#explicit-averaging-operations-250),
|
|
103
|
+
and an opt-in [strict lookup policy for null filtering](fpstreams/docs/reference/rows.md#null-filtering-and-lookup-failures-250).
|
|
104
|
+
These APIs are not in 2.4.1. `Flow.reduce` still returns `Any` to type checkers,
|
|
105
|
+
including with an annotated callback. See the [reduce guide](fpstreams/docs/reference/flow.md#reduce-types)
|
|
106
|
+
for migration examples and [contributor checks](CONTRIBUTING.md#run-the-checks) for validation.
|
|
107
|
+
|
|
78
108
|
## Install
|
|
79
109
|
|
|
80
|
-
Install
|
|
110
|
+
Install fpstreams from PyPI:
|
|
81
111
|
|
|
82
112
|
~~~bash
|
|
83
113
|
python -m pip install fpstreams
|
|
@@ -96,48 +126,47 @@ python -m pip install "fpstreams[data]" # NumPy, pandas, and PyArrow
|
|
|
96
126
|
python -m pip install "fpstreams[polars]" # Polars and PyArrow
|
|
97
127
|
~~~
|
|
98
128
|
|
|
99
|
-
##
|
|
129
|
+
## Quick start
|
|
100
130
|
|
|
101
|
-
|
|
102
|
-
Select the Python interpreter where fpstreams is installed in your editor to get
|
|
103
|
-
method completions, parameter hints, and hover documentation.
|
|
131
|
+
Pass an iterable to `flow()` and consume the result:
|
|
104
132
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
`flow([1, 2, 3])` is inferred as `Flow[int]`.
|
|
133
|
+
~~~python
|
|
134
|
+
from fpstreams import flow
|
|
108
135
|
|
|
109
|
-
|
|
136
|
+
result = flow([1, 2, 3]).filter(lambda value: value > 1).map(lambda value: value * 10).to_list()
|
|
137
|
+
assert result == [20, 30]
|
|
138
|
+
~~~
|
|
139
|
+
|
|
140
|
+
Use [async pipelines](#asynchronous-pipelines) for asynchronous callbacks and
|
|
141
|
+
[record pipelines](#record-pipelines) for grouping. The
|
|
142
|
+
[engine guide](#execution-engines) helps diagnose execution routes;
|
|
143
|
+
[resource limits](#resource-and-file-safety-controls) apply when processing larger inputs.
|
|
110
144
|
|
|
111
|
-
|
|
112
|
-
|
|
145
|
+
### Close an owned iterator
|
|
146
|
+
|
|
147
|
+
This pipeline takes three matching values and closes its generator before returning.
|
|
148
|
+
The generator prints when its `finally` block runs, so cleanup is visible:
|
|
113
149
|
|
|
114
150
|
~~~python
|
|
115
|
-
from fpstreams import
|
|
151
|
+
from fpstreams import flow
|
|
116
152
|
|
|
117
|
-
orders = [
|
|
118
|
-
{"region": "eu", "status": "paid", "amount": 24},
|
|
119
|
-
{"region": "us", "status": "paid", "amount": 20},
|
|
120
|
-
{"region": "eu", "status": "cancelled", "amount": 99},
|
|
121
|
-
{"region": "eu", "status": "paid", "amount": 24},
|
|
122
|
-
]
|
|
123
153
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
orders=agg.count(),
|
|
130
|
-
revenue=agg.sum("amount"),
|
|
131
|
-
)
|
|
132
|
-
.sort_by("region")
|
|
133
|
-
.to_list()
|
|
134
|
-
)
|
|
154
|
+
def numbers():
|
|
155
|
+
try:
|
|
156
|
+
yield from range(10)
|
|
157
|
+
finally:
|
|
158
|
+
print("source closed")
|
|
135
159
|
|
|
160
|
+
|
|
161
|
+
result = flow(numbers()).filter(lambda value: value % 2 == 0).take(3).to_list()
|
|
136
162
|
print(result)
|
|
137
|
-
#
|
|
138
|
-
#
|
|
163
|
+
# source closed
|
|
164
|
+
# [0, 2, 4]
|
|
139
165
|
~~~
|
|
140
166
|
|
|
167
|
+
See [record pipelines](#record-pipelines) for grouped aggregation, or
|
|
168
|
+
[asynchronous pipelines](#asynchronous-pipelines) for concurrent calls.
|
|
169
|
+
|
|
141
170
|
## When to use fpstreams
|
|
142
171
|
|
|
143
172
|
For a short, local transformation, a comprehension is often enough. fpstreams
|
|
@@ -173,10 +202,11 @@ standard Arrow/dataframe protocol routing, retained NumPy execution, and new
|
|
|
173
202
|
async queue, prefetch, window, and numeric terminal APIs. See the
|
|
174
203
|
[changelog](https://github.com/steventimes/fpstreams/blob/master/CHANGELOG.md) for the release summary.
|
|
175
204
|
|
|
176
|
-
|
|
177
|
-
CPython 3.14t
|
|
178
|
-
|
|
179
|
-
|
|
205
|
+
The CI matrix targets standard CPython 3.11 through 3.15.
|
|
206
|
+
Free-threaded CPython 3.14t and 3.15t are included in an
|
|
207
|
+
experimental, non-blocking CI matrix that builds the native extension and checks
|
|
208
|
+
core concurrency contracts. These builds are not release-wheel targets;
|
|
209
|
+
optional adapter compatibility is outside this matrix.
|
|
180
210
|
|
|
181
211
|
### Value pipelines
|
|
182
212
|
|
|
@@ -274,7 +304,9 @@ record column. Explicit Flow factories cover Arrow, equal-length named columns,
|
|
|
274
304
|
NumPy arrays, dataframe, Polars, typed CSV, and Parquet inputs. The `rows`
|
|
275
305
|
namespace continues to provide compatibility CSV, JSONL, SQLite, DB-API, and
|
|
276
306
|
record-oriented output methods. CSV and JSONL
|
|
277
|
-
inputs accept paths, caller-owned open handles, or replayable opener functions
|
|
307
|
+
inputs accept paths, caller-owned open handles, or replayable opener functions.
|
|
308
|
+
The following example needs `fpstreams[arrow]` and an `accounts.parquet` file
|
|
309
|
+
with `id`, `status`, and `balance` columns:
|
|
278
310
|
|
|
279
311
|
~~~python
|
|
280
312
|
from fpstreams import col, flow
|
|
@@ -323,6 +355,8 @@ asyncio.run(main())
|
|
|
323
355
|
|
|
324
356
|
Async iterators consumed by a pipeline are closed, and tasks created by the
|
|
325
357
|
pipeline are cancelled, when it finishes, errors, times out, or short-circuits.
|
|
358
|
+
Mappers must cooperate with cancellation so cleanup can finish. Offload blocking
|
|
359
|
+
work explicitly rather than running it directly inside an async mapper.
|
|
326
360
|
|
|
327
361
|
### Key/value pipelines
|
|
328
362
|
|
|
@@ -365,8 +399,13 @@ assert plan["data_movement"] == {
|
|
|
365
399
|
You can request an engine explicitly when testing parity or diagnosing a plan:
|
|
366
400
|
|
|
367
401
|
~~~python
|
|
402
|
+
from fpstreams import flow, item
|
|
403
|
+
|
|
404
|
+
pipeline = flow([1, 2, 3]).map(item * 2)
|
|
368
405
|
python_result = pipeline.with_engine("python").to_list()
|
|
369
406
|
native_result = pipeline.with_engine("native").to_list()
|
|
407
|
+
|
|
408
|
+
assert python_result == native_result == [2, 4, 6]
|
|
370
409
|
~~~
|
|
371
410
|
|
|
372
411
|
Use `run_with_report()` to get a terminal's value, recorded route, and query-owned
|
|
@@ -416,11 +455,23 @@ finish. The defaults are 100,000 rows and 64 MiB per partition, 100,000 matches
|
|
|
416
455
|
per key, 1,000,000 output rows, and three repartition levels.
|
|
417
456
|
|
|
418
457
|
~~~python
|
|
419
|
-
from fpstreams import SpillLimits, rows
|
|
458
|
+
from fpstreams import SpillLimits, agg, rows
|
|
459
|
+
|
|
460
|
+
accounts = [{"id": 1, "account_id": "a"}, {"id": 2, "account_id": "a"}]
|
|
461
|
+
balances = [{"id": 1, "balance": 10}, {"id": 2, "balance": 20}]
|
|
420
462
|
|
|
421
463
|
limits = SpillLimits(max_output_rows=250_000, max_matches_per_key=10_000)
|
|
422
|
-
|
|
423
|
-
grouped =
|
|
464
|
+
joined = rows(accounts).join(balances, on="id", partitions=32, limits=limits).to_list()
|
|
465
|
+
grouped = (
|
|
466
|
+
rows(accounts)
|
|
467
|
+
.group_by("account_id")
|
|
468
|
+
.spill(partitions=32, limits=limits)
|
|
469
|
+
.aggregate(count=agg.count())
|
|
470
|
+
.to_list()
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
assert sorted((row["id"], row["balance"]) for row in joined) == [(1, 10), (2, 20)]
|
|
474
|
+
assert grouped == [{"account_id": "a", "count": 2}]
|
|
424
475
|
~~~
|
|
425
476
|
|
|
426
477
|
Exceeding a configured partition, match, state, record, or output limit raises
|
|
@@ -445,8 +496,31 @@ only for trusted local input when an unlimited record is intentional.
|
|
|
445
496
|
once during construction and treated as one-shot; a PyArrow
|
|
446
497
|
`RecordBatchReader` is also one-shot.
|
|
447
498
|
- `flow.defer(factory)` opens a fresh source for every execution.
|
|
448
|
-
- Terminal operations
|
|
449
|
-
|
|
499
|
+
- Terminal operations release owned iterators, database cursors, and temporary
|
|
500
|
+
files, including on errors and early termination. Async execution also cancels
|
|
501
|
+
and awaits tasks it creates. File adapters distinguish owned resources from
|
|
502
|
+
caller-owned handles; see the [I/O reference](https://steventimes.github.io/fpstreams/reference/io/).
|
|
503
|
+
|
|
504
|
+
## Editor support
|
|
505
|
+
|
|
506
|
+
fpstreams includes inline type annotations, docstrings, and the `py.typed` marker.
|
|
507
|
+
Select the Python interpreter where fpstreams is installed in your editor to get
|
|
508
|
+
method completions, parameter hints, and hover documentation.
|
|
509
|
+
|
|
510
|
+
Version 2.2.1 fixes `rows` factory completion and keeps core
|
|
511
|
+
iterable element types when optional data adapters are absent. For example,
|
|
512
|
+
`flow([1, 2, 3])` is inferred as `Flow[int]`.
|
|
513
|
+
|
|
514
|
+
## Coding-agent guidance
|
|
515
|
+
|
|
516
|
+
The [Flow](fpstreams/docs/reference/flow.md#reduce-types),
|
|
517
|
+
[async Flow](fpstreams/docs/reference/async_flow.md#explicit-asynchronous-accumulator-contract-250),
|
|
518
|
+
and [Collector](fpstreams/docs/reference/collecting.md#explicit-averaging-operations-250)
|
|
519
|
+
references describe entry points, typed reductions, and migration contracts.
|
|
520
|
+
An independently supplied fpstreams usage skill must match the installed library
|
|
521
|
+
version and its API documentation. The skill is distributed separately from
|
|
522
|
+
the PyPI wheel and sdist. Neither a package install nor its presence in a
|
|
523
|
+
checkout registers it with an agent.
|
|
450
524
|
|
|
451
525
|
## v1 compatibility
|
|
452
526
|
|
|
@@ -470,25 +544,20 @@ uv sync --extra arrow --extra data --extra polars \
|
|
|
470
544
|
uv run maturin develop --release
|
|
471
545
|
uv run pytest -W error --cov=src/fpstreams --cov-branch --cov-report=term-missing
|
|
472
546
|
uv run coverage report
|
|
473
|
-
uv run ruff check src tests scripts
|
|
474
|
-
uv run ruff format --check src tests scripts
|
|
547
|
+
uv run ruff check src tests scripts
|
|
548
|
+
uv run ruff format --check src tests scripts
|
|
475
549
|
uv run mypy src/fpstreams
|
|
476
550
|
cargo test --manifest-path rust/Cargo.toml
|
|
477
551
|
uv run python scripts/build_browser_wheel.py
|
|
478
552
|
uv run mkdocs build --strict -f fpstreams/mkdocs.yml
|
|
479
553
|
~~~
|
|
480
554
|
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
filters, and optional JSON output. Competitive mode checks equivalent tasks
|
|
484
|
-
against Python, NumPy, and pandas and prints the percentage difference.
|
|
555
|
+
Performance measurements are maintained locally. Benchmark tools and raw reports
|
|
556
|
+
are not part of the public source distribution or GitHub Actions.
|
|
485
557
|
|
|
486
558
|
The source tree is organized by domain under `src/fpstreams/`: `streams`,
|
|
487
559
|
`planning`, `execution`, `collecting`, `tabular`, `expressions`, and `primitives`.
|
|
488
560
|
Small top-level modules are compatibility facades, not duplicate implementations.
|
|
489
|
-
The [Chinese code-reading guide](https://github.com/steventimes/fpstreams/blob/master/CODE_READING_GUIDE.zh-CN.md)
|
|
490
|
-
walks through the main sync, relational, spill, native, and async execution paths
|
|
491
|
-
in those domains.
|
|
492
561
|
|
|
493
562
|
Contributions are welcome. The
|
|
494
563
|
[contributing guide](https://github.com/steventimes/fpstreams/blob/master/CONTRIBUTING.md)
|