fpstreams 2.4.1__tar.gz → 2.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fpstreams-2.4.1 → fpstreams-2.5.0}/CHANGELOG.md +214 -2
- {fpstreams-2.4.1 → fpstreams-2.5.0}/PKG-INFO +122 -54
- {fpstreams-2.4.1 → fpstreams-2.5.0}/README.md +116 -53
- {fpstreams-2.4.1 → fpstreams-2.5.0}/pyproject.toml +28 -5
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/Cargo.lock +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/Cargo.toml +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/common.rs +12 -2
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/float/endpoints.rs +34 -35
- fpstreams-2.5.0/rust/src/float/rolling.rs +155 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/float.rs +3 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/integer/endpoints.rs +92 -15
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/integer.rs +2 -2
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/lib.rs +8 -7
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/numeric/buffers.rs +47 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/numeric/reductions.rs +70 -2
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/__init__.py +12 -12
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/_native.pyi +6 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/aggregate.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/async_flow.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/collecting/_collector_base.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/collecting/aggregate_program.py +1 -6
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/collecting/aggregation.py +24 -7
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/collecting/collector.py +533 -47
- fpstreams-2.5.0/src/fpstreams/collecting/sketches.py +122 -0
- fpstreams-2.5.0/src/fpstreams/collecting/statistics.py +383 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/collectors.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/column.py +1 -1
- fpstreams-2.5.0/src/fpstreams/errors.py +72 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/exceptions.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/_rows_fusion.py +3 -7
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/_scalar_fusion.py +4 -15
- fpstreams-2.4.1/src/fpstreams/execution/arrow.py → fpstreams-2.5.0/src/fpstreams/execution/arrow/__init__.py +107 -371
- fpstreams-2.5.0/src/fpstreams/execution/arrow/numeric.py +335 -0
- fpstreams-2.5.0/src/fpstreams/execution/arrow/sort.py +129 -0
- fpstreams-2.4.1/src/fpstreams/execution/async_scheduler.py → fpstreams-2.5.0/src/fpstreams/execution/asynchronous/__init__.py +85 -17
- fpstreams-2.5.0/src/fpstreams/execution/asynchronous/concurrent.py +288 -0
- fpstreams-2.4.1/src/fpstreams/execution/async_merge.py → fpstreams-2.5.0/src/fpstreams/execution/asynchronous/merge.py +16 -8
- fpstreams-2.4.1/src/fpstreams/execution/async_iterators.py → fpstreams-2.5.0/src/fpstreams/execution/asynchronous/operators.py +119 -14
- fpstreams-2.4.1/src/fpstreams/execution/async_timers.py → fpstreams-2.5.0/src/fpstreams/execution/asynchronous/timers.py +9 -9
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/native.py +2 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/numpy_group.py +6 -18
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/numpy_prefix.py +1 -4
- fpstreams-2.4.1/src/fpstreams/execution/_pair_aggregate.py → fpstreams-2.5.0/src/fpstreams/execution/pairs/__init__.py +2 -2
- fpstreams-2.4.1/src/fpstreams/execution/_pair_dict.py → fpstreams-2.5.0/src/fpstreams/execution/pairs/dictionary.py +43 -32
- fpstreams-2.4.1/src/fpstreams/execution/_pair_row_filter.py → fpstreams-2.5.0/src/fpstreams/execution/pairs/row_filter.py +13 -17
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/physical.py +7 -7
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/relational/__init__.py +8 -16
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/relational/arrow_global.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/relational/arrow_group.py +3 -2
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/relational/collector_group.py +8 -9
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/relational/join.py +8 -7
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/relational/native_group.py +3 -8
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/sync.py +10 -3
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/sync_ops.py +81 -26
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expr.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/_codegen.py +20 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/_row_codegen.py +6 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/program.py +29 -26
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/row.py +19 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/row_eval.py +17 -5
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/selectors.py +67 -20
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/typed_ir.py +5 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/gather.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/io_safety.py +12 -3
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/option.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/physical/compiled.py +1 -3
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/physical/kernel_cache.py +3 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/physical/plan.py +2 -0
- fpstreams-2.4.1/src/fpstreams/planning/arrow.py → fpstreams-2.5.0/src/fpstreams/planning/arrow/__init__.py +24 -12
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/async_.py +109 -11
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/async_utils.py +3 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/compiler.py +30 -32
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/explain.py +21 -6
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/gather.py +5 -1
- fpstreams-2.5.0/src/fpstreams/planning/lineage.py +308 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/logical.py +20 -2
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/native.py +7 -7
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/numpy.py +1 -1
- fpstreams-2.4.1/src/fpstreams/planning/pair_i64_expression.py → fpstreams-2.5.0/src/fpstreams/planning/pairs/i64_expression.py +32 -8
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/plan_cache.py +2 -0
- fpstreams-2.4.1/src/fpstreams/planning/relational.py → fpstreams-2.5.0/src/fpstreams/planning/relational/__init__.py +13 -17
- fpstreams-2.4.1/src/fpstreams/planning/relational_specs.py → fpstreams-2.5.0/src/fpstreams/planning/relational/specs.py +13 -33
- fpstreams-2.4.1/src/fpstreams/planning/semantics.py → fpstreams-2.5.0/src/fpstreams/planning/semantics/__init__.py +6 -1
- fpstreams-2.4.1/src/fpstreams/planning/semantic_rules.py → fpstreams-2.5.0/src/fpstreams/planning/semantics/analysis.py +301 -33
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/source.py +3 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/sync.py +13 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/result.py +1 -1
- fpstreams-2.5.0/src/fpstreams/runtime/_distinct.py +124 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/iterators.py +58 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/query.py +8 -2
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/resources.py +30 -26
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/tasks.py +49 -17
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/storage/checkpoint.py +34 -14
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/storage/spill_store.py +3 -2
- fpstreams-2.4.1/src/fpstreams/streams/_flow_materialize.py → fpstreams-2.5.0/src/fpstreams/streams/_terminals/__init__.py +53 -36
- fpstreams-2.4.1/src/fpstreams/streams/_flow_numeric.py → fpstreams-2.5.0/src/fpstreams/streams/_terminals/reductions.py +204 -30
- fpstreams-2.4.1/src/fpstreams/streams/_flow_structural_list.py → fpstreams-2.5.0/src/fpstreams/streams/_terminals/structural.py +48 -45
- fpstreams-2.4.1/src/fpstreams/streams/_flow_unique_list.py → fpstreams-2.5.0/src/fpstreams/streams/_terminals/unique.py +36 -19
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/streams/async_flow.py +149 -25
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/streams/async_terminals.py +69 -20
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/streams/flow.py +65 -21
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/streams/flow_terminals.py +93 -89
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/streams/pairs.py +46 -19
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/arrow.py +2 -6
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/dataframe.py +10 -6
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/factory.py +5 -7
- fpstreams-2.4.1/src/fpstreams/tabular/io.py → fpstreams-2.5.0/src/fpstreams/tabular/io/__init__.py +38 -55
- fpstreams-2.5.0/src/fpstreams/tabular/io/database.py +559 -0
- fpstreams-2.4.1/src/fpstreams/tabular/_text_sources.py → fpstreams-2.5.0/src/fpstreams/tabular/io/text.py +25 -12
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/join.py +8 -11
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/numpy.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/polars.py +8 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/rows.py +82 -82
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/spill.py +6 -21
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/spill_io.py +1 -1
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/testing.py +19 -6
- fpstreams-2.4.1/src/fpstreams/collecting/statistics.py +0 -194
- fpstreams-2.4.1/src/fpstreams/errors.py +0 -33
- fpstreams-2.4.1/src/fpstreams/execution/async_map.py +0 -92
- fpstreams-2.4.1/src/fpstreams/execution/async_ops.py +0 -97
- fpstreams-2.4.1/src/fpstreams/execution/async_prefetch.py +0 -105
- fpstreams-2.4.1/src/fpstreams/execution/async_queue.py +0 -94
- fpstreams-2.4.1/src/fpstreams/planning/semantic_analyzer.py +0 -168
- fpstreams-2.4.1/src/fpstreams/runtime/_distinct.py +0 -35
- fpstreams-2.4.1/src/fpstreams/streams/_flow_frequencies.py +0 -203
- fpstreams-2.4.1/src/fpstreams/tabular/sql.py +0 -236
- fpstreams-2.4.1/src/fpstreams/tabular/sqlite_sink.py +0 -381
- {fpstreams-2.4.1 → fpstreams-2.5.0}/LICENSE +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/build.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/float/affine_pair.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/numeric_mean.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/numpy_export.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/numpy_group/buffer.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/numpy_group.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/pair/expr.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/pair/prefix.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/pair/row_filter.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/pair/unique.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/pair/value_filter.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/pair/value_map.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/pair.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/pivot.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/records.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/adapters/namedtuple.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/adapters.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/global_numeric.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/group_numeric.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/group_pair_expr.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/join_callable/many.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/join_callable/unique.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/join_callable.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/join_exact.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational/join_i64.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational_fixed/composite.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational_fixed/global_multi/same_field.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational_fixed/global_multi.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational_fixed/group_multi.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational_fixed/single.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/relational_fixed.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/scalar_sort.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/scalar_unique.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/select.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/adapters.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/group_exact/pairs.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/group_exact/rows.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/group_exact.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/group_fixed.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/join_callable/behavior.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/join_callable/direct.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/join_callable.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/join_exact.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/numeric/kernels.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/numeric.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests/numpy_group.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/tests.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/unnest.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/rust/src/unpivot.rs +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/_provenance.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/collecting/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/collecting/program.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/collecting/reducer.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/relational/arrow_group_rows.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/sorted_streams.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/sorting.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/execution/window_reduce.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/row_ir.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/expressions/scalar.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/functional.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/physical/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/physical/async_plan.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/physical/relational.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/planning/__init__.py +0 -0
- /fpstreams-2.4.1/src/fpstreams/planning/arrow_source.py → /fpstreams-2.5.0/src/fpstreams/planning/arrow/source.py +0 -0
- /fpstreams-2.4.1/src/fpstreams/planning/_pair_stages.py → /fpstreams-2.5.0/src/fpstreams/planning/pairs/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/primitives/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/primitives/option.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/primitives/result.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/py.typed +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/executor.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/failpoints.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/files.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/limits.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/metrics.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/report.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/runtime/spill.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/storage/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/storage/codec.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/streams/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/streams/_async_identity.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/__init__.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/grouped.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/records.py +0 -0
- {fpstreams-2.4.1 → fpstreams-2.5.0}/src/fpstreams/tabular/spill_limits.py +0 -0
|
@@ -1,9 +1,221 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
This file records user-visible and compatibility-relevant changes in fpstreams
|
|
3
|
+
This file records user-visible and compatibility-relevant changes in fpstreams,
|
|
4
4
|
including changed defaults.
|
|
5
5
|
|
|
6
|
-
##
|
|
6
|
+
## 2.5.0
|
|
7
|
+
|
|
8
|
+
Version 2.5.0 includes an intentional static typing incompatibility: `Collectors.averaging()` now
|
|
9
|
+
returns `object` in type annotations rather than incorrectly promising `float`.
|
|
10
|
+
Runtime arithmetic is unchanged. Use `Collectors.averaging_with()` for explicit
|
|
11
|
+
accumulator and result types, or narrow the legacy result before use.
|
|
12
|
+
|
|
13
|
+
Dynamic `reduce()` remains dynamically typed. For an explicit accumulator
|
|
14
|
+
contract, use an annotated `Flow.fold()` result or `AsyncFlow.fold_async()`.
|
|
15
|
+
The migration examples are in the collecting and async flow reference pages;
|
|
16
|
+
2.5.0 does not claim to solve automatic `reduce()` inference.
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- `AsyncFlow.fold_async()` provides an explicit asynchronous state contract.
|
|
21
|
+
Use annotated `Flow.fold()` or `AsyncFlow.fold_async()` results when migrating
|
|
22
|
+
dynamic `reduce()` calls to a declared accumulator type.
|
|
23
|
+
- `Collectors.averaging_with()` accepts typed accumulation, division, and empty
|
|
24
|
+
result callbacks for numeric types and custom arithmetic.
|
|
25
|
+
- `Rows.drop_nulls(missing="raise")` propagates lookup failures. Callable
|
|
26
|
+
selectors can define actual absence by returning `None`; the default
|
|
27
|
+
`missing="legacy"` preserves historical lookup handling.
|
|
28
|
+
- `Collectors.averaging()` now declares its result as `object`, replacing the
|
|
29
|
+
inaccurate `float` annotation. Runtime arithmetic is unchanged; migrate typed
|
|
30
|
+
consumers to `averaging_with()` or narrow the result explicitly.
|
|
31
|
+
- `Flow.chunk()` and `AsyncFlow.chunk()` accept `strict=True` to reject an
|
|
32
|
+
incomplete final batch during consumption. The `batch` aliases also support it.
|
|
33
|
+
- `Collectors.frequencies()` counts hashable inputs in first-encounter order and
|
|
34
|
+
composes with synchronous, asynchronous, and grouped collectors.
|
|
35
|
+
- `Collectors.mode()` finds the most frequent hashable value in sync, async,
|
|
36
|
+
and grouped collection, with first-encountered tie breaking.
|
|
37
|
+
- Extend experimental free-threaded core-concurrency CI to Python 3.15t alongside
|
|
38
|
+
3.14t. Check that native import keeps the GIL disabled, and keep optional adapter
|
|
39
|
+
dependencies out of these jobs through the shared `test-core` group.
|
|
40
|
+
- Add standard CPython 3.15 to the runtime and consumer typing CI matrices.
|
|
41
|
+
The frozen data-adapter environment now uses pandas 3.0.6 and PyArrow 26.0.0
|
|
42
|
+
with Python 3.15 wheels.
|
|
43
|
+
- Runnable resource-ownership examples cover early termination, callback errors,
|
|
44
|
+
and caller cancellation. CI executes their documented code without optional extras.
|
|
45
|
+
- `Collectors.statistics(selector, ddof=0)` computes count, mean, variance, and
|
|
46
|
+
standard deviation with one selection and conversion per valid item.
|
|
47
|
+
- Optional `fpstreams[sketch]` collectors provide KLL approximate quantiles and
|
|
48
|
+
HLL approximate distinct counts through the existing sync, async, and grouped
|
|
49
|
+
reduction interfaces. Exact collectors keep their existing behavior.
|
|
50
|
+
- `Flow.rolling_statistics(size, selector=..., ddof=0)` and its async equivalent
|
|
51
|
+
emit count, mean, and variance for recent items, including warmup windows.
|
|
52
|
+
They reuse `map_accum`, with independent state for every evaluation.
|
|
53
|
+
- `Rows.explain().to_dict(include_field_dependencies=True)` exports static field dependencies.
|
|
54
|
+
Direct projections, computed columns, renames, and supported relational
|
|
55
|
+
operations retain source identities; unsupported operations report unknown
|
|
56
|
+
dependencies. Value inputs and row selection inputs are reported separately.
|
|
57
|
+
|
|
58
|
+
### Performance
|
|
59
|
+
|
|
60
|
+
- Use C-backed batching for eligible strict retained-list chunks and exact
|
|
61
|
+
generator, map, and filter cursors on Python 3.12+. Python 3.11 and custom
|
|
62
|
+
cursors retain their iterator path.
|
|
63
|
+
- Reduce intermediate generator calls in legacy Gatherers and selected-field
|
|
64
|
+
`Rows.drop_nulls()` while preserving lookup errors and short-circuiting.
|
|
65
|
+
- Use an optional Rust numeric window state for rolling statistics. Selection
|
|
66
|
+
and numeric conversion remain in Python. Explicit Python execution and
|
|
67
|
+
extensions without this capability retain the reference implementation.
|
|
68
|
+
|
|
69
|
+
### Fixed
|
|
70
|
+
|
|
71
|
+
- Preserve the original custom-key hash exception in synchronous and asynchronous
|
|
72
|
+
per-key reductions, frequency counts, Pair collection, and in-memory Python
|
|
73
|
+
joins and grouping on CPython 3.14 and 3.15. Keep hash calls, raw key identity, and source cleanup unchanged.
|
|
74
|
+
|
|
75
|
+
- Reuse each compiled scalar stage's validated fingerprint for the physical-plan
|
|
76
|
+
cache instead of encoding the same expression again. Row expressions remain
|
|
77
|
+
outside structural caches.
|
|
78
|
+
|
|
79
|
+
- NumPy filter planning checks numeric classes by identity, preserving Python
|
|
80
|
+
fallback for custom integer types whose metaclasses define hashing or equality.
|
|
81
|
+
- Materializing deep `RowExpr` maps and filters skips the unused structural
|
|
82
|
+
fingerprint and reaches the iterative evaluator when direct compilation is bounded.
|
|
83
|
+
- Row-expression effect analysis visits shared IR nodes once. Evaluation still
|
|
84
|
+
executes each occurrence, including repeated selectors and callbacks.
|
|
85
|
+
|
|
86
|
+
- Close the iterator returned by custom Rows/Flow iteration in `to_columns()`
|
|
87
|
+
after exhaustion or conversion failure, preserving the primary error.
|
|
88
|
+
- Respect custom row iteration in `arrow_batches()` with default or explicit
|
|
89
|
+
schemas. Check iteration at consumption time while retaining eligible native
|
|
90
|
+
paths and bounded batch conversion.
|
|
91
|
+
|
|
92
|
+
- Finish a cancelled child's cleanup before propagating cancellation of its drain
|
|
93
|
+
waiter. Repeated caller cancellation no longer interrupts that cleanup or lets
|
|
94
|
+
timeout operators close the source before its pending pull has stopped.
|
|
95
|
+
|
|
96
|
+
- Preserve each result when a shared row-expression node is evaluated more than
|
|
97
|
+
once. Stateful callbacks and dynamic selectors now agree with direct evaluation
|
|
98
|
+
in the iterative fallback, including nested lazy operands.
|
|
99
|
+
|
|
100
|
+
- Convert Arrow extension columns on pandas 2.0 without the dataframe interchange
|
|
101
|
+
protocol when copying is allowed. Keep conversion lazy and exclude index columns.
|
|
102
|
+
- Preserve zero-column row counts in Polars output, including bounded batches,
|
|
103
|
+
on the declared minimum Polars version.
|
|
104
|
+
|
|
105
|
+
- Keep generated row and scalar functions compatible with Python 3.11 AST types.
|
|
106
|
+
|
|
107
|
+
- Reject NaN async durations before opening a source or starting work. Zero stays
|
|
108
|
+
valid for `interval` and `debounce`. An invalid timer rearm preserves its
|
|
109
|
+
existing timer task.
|
|
110
|
+
|
|
111
|
+
- Normalize integer counts for `merge_map` concurrency and synchronous parallel
|
|
112
|
+
workers/buffers before execution. Noninteger values raise `TypeError`, and later
|
|
113
|
+
mutation of an integer-like configuration object cannot change the stored limit.
|
|
114
|
+
|
|
115
|
+
- Require integer capacities for plan and kernel caches, normalizing integer-like
|
|
116
|
+
values once so later mutation cannot change the configured bound.
|
|
117
|
+
|
|
118
|
+
- Accept inner async iterators without an `aclose` hook in `merge_map` and
|
|
119
|
+
`switch_map`. Optional hooks may return an awaitable or a synchronous value;
|
|
120
|
+
cleanup retains the hook selected when the inner opens.
|
|
121
|
+
|
|
122
|
+
- Close the owned outer source when `switch_map` cannot submit its first pull
|
|
123
|
+
task, retaining the submission error and any source cleanup diagnostics.
|
|
124
|
+
Root ownership also covers unstarted intermediate stages and sync-to-async
|
|
125
|
+
adapters without consuming an item or repeating source close.
|
|
126
|
+
|
|
127
|
+
- Drain existing scope tasks before closing upstream resources when a task factory
|
|
128
|
+
rejects the cleanup task. The original submission error remains observable.
|
|
129
|
+
|
|
130
|
+
- Preserve JSON Lines record boundaries when disabling the byte limit. Managed
|
|
131
|
+
paths now use LF/CRLF framing in both modes; borrowed text handles retain the
|
|
132
|
+
caller's newline handling.
|
|
133
|
+
|
|
134
|
+
- Require an initial BOM for nonempty JSON Lines input using generic UTF-16/32
|
|
135
|
+
codecs, consistently across byte-limit modes. Explicit byte-order codecs
|
|
136
|
+
continue to support data without a BOM.
|
|
137
|
+
|
|
138
|
+
- Handle UTF-8 signatures consistently across bounded and unbounded JSON Lines
|
|
139
|
+
sources. A signature before a later record is no longer silently discarded.
|
|
140
|
+
|
|
141
|
+
- Wait for in-flight asynchronous file operations before completing cancellation
|
|
142
|
+
or closing their handles, including cancellation during file acquisition.
|
|
143
|
+
|
|
144
|
+
- Preserve asynchronous file read errors and caller cancellation when file close
|
|
145
|
+
also fails. Close failures still propagate after exhaustion or early termination.
|
|
146
|
+
|
|
147
|
+
- Preserve serialization errors when text writer and source cleanup also fail.
|
|
148
|
+
Atomic CSV, JSON, and JSON Lines exports keep the previous destination on failure.
|
|
149
|
+
|
|
150
|
+
- Preserve checkpoint callback failures when staging cleanup or lock-file close also
|
|
151
|
+
fails. Retries verify and reuse entries published before a later cleanup error.
|
|
152
|
+
|
|
153
|
+
- Clarify how asynchronous `reduce`, `reduce_right`, and `fold_right` resolve
|
|
154
|
+
reducer results while preserving initial values and singleton items unchanged.
|
|
155
|
+
|
|
156
|
+
- Preserve operation errors and continue closing owned resources when a cleanup
|
|
157
|
+
exception raises while formatting its message.
|
|
158
|
+
|
|
159
|
+
- Preserve element and fallback types for synchronous and asynchronous `first`,
|
|
160
|
+
`last`, `find`, and `nth` results. Calls without a default retain the element type.
|
|
161
|
+
|
|
162
|
+
- Retain annotated container Collector results for `Any` input elements and
|
|
163
|
+
infer awaited callable results in async collection. Optional-collector
|
|
164
|
+
forwarding and named results keep their existing contracts.
|
|
165
|
+
- Preserve inferred Collector element/result types for direct container factories
|
|
166
|
+
while retaining explicit `Collectors[int]` construction. Per-key collection
|
|
167
|
+
keeps the collector state type; input variance reflects consumption.
|
|
168
|
+
- Reject non-real numeric scalars such as NumPy `complex64` before converting
|
|
169
|
+
statistical inputs or quantile ranks to floats. Decimal, Fraction, and real
|
|
170
|
+
NumPy scalar inputs keep their existing support.
|
|
171
|
+
- Concurrent async mapping can emit completed results while the source is still
|
|
172
|
+
waiting to produce another item. With concurrency above one, a reader task
|
|
173
|
+
also counts against the query's task limit.
|
|
174
|
+
- `map_async()` normalizes `buffer` through the integer-index protocol when the
|
|
175
|
+
pipeline is built, rejecting noninteger bounds before opening the source.
|
|
176
|
+
- Python and Rust variance calculations retain small differences in data with a
|
|
177
|
+
large common offset. Existing nonfinite-input policies remain unchanged.
|
|
178
|
+
- `Flow.to_async()` preserves its item type, and `Flow.pairs()` preserves tuple
|
|
179
|
+
key and value types. Dynamic pair inputs keep their existing runtime support.
|
|
180
|
+
- Native integer and floating-point range endpoints periodically check Python
|
|
181
|
+
signals during long scans.
|
|
182
|
+
- Reuse native range bounds for periodic signal checks, avoiding a separate
|
|
183
|
+
per-item countdown while preserving the 4096-input check interval.
|
|
184
|
+
|
|
185
|
+
### Changed
|
|
186
|
+
|
|
187
|
+
- Retained primitive Arrow sources with a complete compatible columnar prefix
|
|
188
|
+
collect directly into `to_list()`, preserving conversion and cleanup hooks.
|
|
189
|
+
- Optional field-dependency export reuses each computed-column analysis within
|
|
190
|
+
its step while preserving opaque-expression short circuiting.
|
|
191
|
+
|
|
192
|
+
- Keep performance runners and their regression fixtures local to maintainers.
|
|
193
|
+
Public source distributions and CI retain library checks without benchmark
|
|
194
|
+
components or hosted timing jobs.
|
|
195
|
+
- Consolidate internal async execution, pair execution, semantic analysis,
|
|
196
|
+
relational planning, terminal helpers, Arrow planning/execution, pair planning,
|
|
197
|
+
and I/O into domain packages. Public
|
|
198
|
+
imports remain unchanged; private implementation paths have moved.
|
|
199
|
+
- Release excess native numeric vector capacity before constructing list, tuple,
|
|
200
|
+
or set results, reducing overlapping temporary storage.
|
|
201
|
+
- Read row-stage metadata without a shared registry lock. Registration and
|
|
202
|
+
reclamation remain synchronized, with weak owner and executable-code checks.
|
|
203
|
+
- Reuse expression analysis within each field-dependency export. Spill partition
|
|
204
|
+
writes consume their existing byte buffers without an intermediate copy.
|
|
205
|
+
- Guarded Arrow projections can retain builtin scalar and list columns after
|
|
206
|
+
full buffer validation. Unsupported nested types retain row conversion.
|
|
207
|
+
- Plan reasons distinguish representation selection from terminal execution.
|
|
208
|
+
- Separate Arrow numeric reductions and retained sorting from batch dispatch,
|
|
209
|
+
preserving the existing execution entry points and live callback bindings.
|
|
210
|
+
- Document the existing v2 `compact()` difference between Flow and AsyncFlow;
|
|
211
|
+
use `filter_none()` to retain falsey values other than `None` in both domains.
|
|
212
|
+
- Correct CSV scan documentation: query projection follows conversion of all
|
|
213
|
+
caller-visible columns. Explicit Arrow conversion options can restrict the
|
|
214
|
+
reader's columns.
|
|
215
|
+
- Release qualification reuses CI before building distributions. New Python
|
|
216
|
+
3.11 adapter jobs exercise `lowest-direct` and `highest` dependency resolutions;
|
|
217
|
+
Windows and macOS smoke selections include async cancellation and executor
|
|
218
|
+
ownership paths. Hosted qualification results remain to be established.
|
|
7
219
|
|
|
8
220
|
## 2.4.1 - 2026-10-18
|
|
9
221
|
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fpstreams
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Programming Language :: Python :: 3
|
|
6
6
|
Classifier: Programming Language :: Python :: 3.11
|
|
7
7
|
Classifier: Programming Language :: Python :: 3.12
|
|
8
8
|
Classifier: Programming Language :: Python :: 3.13
|
|
9
9
|
Classifier: Programming Language :: Python :: 3.14
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
10
11
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
11
12
|
Classifier: License :: OSI Approved :: MIT License
|
|
12
13
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
13
14
|
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: typing-extensions>=4.12 ; python_full_version < '3.13'
|
|
16
|
+
Requires-Dist: datasketches>=5.2,<6 ; extra == 'all'
|
|
14
17
|
Requires-Dist: hypothesis>=6.138,<7 ; extra == 'all'
|
|
15
18
|
Requires-Dist: pytest>=9.0.3,<10 ; extra == 'all'
|
|
16
19
|
Requires-Dist: pytest-cov>=7,<8 ; extra == 'all'
|
|
@@ -27,6 +30,7 @@ Requires-Dist: pandas>=2 ; extra == 'data'
|
|
|
27
30
|
Requires-Dist: pyarrow>=16 ; extra == 'data'
|
|
28
31
|
Requires-Dist: polars>=1.34 ; extra == 'polars'
|
|
29
32
|
Requires-Dist: pyarrow>=16 ; extra == 'polars'
|
|
33
|
+
Requires-Dist: datasketches>=5.2,<6 ; extra == 'sketch'
|
|
30
34
|
Requires-Dist: hypothesis>=6.138,<7 ; extra == 'test'
|
|
31
35
|
Requires-Dist: pytest>=9.0.3,<10 ; extra == 'test'
|
|
32
36
|
Requires-Dist: pytest-cov>=7,<8 ; extra == 'test'
|
|
@@ -40,6 +44,7 @@ Provides-Extra: arrow
|
|
|
40
44
|
Provides-Extra: async
|
|
41
45
|
Provides-Extra: data
|
|
42
46
|
Provides-Extra: polars
|
|
47
|
+
Provides-Extra: sketch
|
|
43
48
|
Provides-Extra: test
|
|
44
49
|
License-File: LICENSE
|
|
45
50
|
Summary: Lazy Python pipelines for iterables and records, with bounded async concurrency.
|
|
@@ -53,21 +58,33 @@ Project-URL: Changelog, https://github.com/steventimes/fpstreams/blob/master/CHA
|
|
|
53
58
|
Project-URL: Documentation, https://steventimes.github.io/fpstreams/
|
|
54
59
|
Project-URL: Homepage, https://github.com/steventimes/fpstreams
|
|
55
60
|
|
|
61
|
+

|
|
62
|
+
|
|
56
63
|
# fpstreams
|
|
57
64
|
|
|
65
|
+
[Install](#install) · [Quick start](#quick-start) · [Examples](#what-is-in-v2) · [Engines](#execution-engines) · [Documentation](https://steventimes.github.io/fpstreams/)
|
|
66
|
+
|
|
67
|
+
| Work with | Start with | Execution |
|
|
68
|
+
| --- | --- | --- |
|
|
69
|
+
| Values and records | `flow()` | Lazy synchronous pipelines |
|
|
70
|
+
| Async inputs and calls | `aflow()` | Bounded concurrent work |
|
|
71
|
+
| Key/value data | `pairs()` | Per-key collection and aggregation |
|
|
72
|
+
|
|
58
73
|
[](https://github.com/steventimes/fpstreams/actions/workflows/ci.yml)
|
|
59
74
|
[](https://pypi.org/project/fpstreams/)
|
|
60
75
|
[](https://pypi.org/project/fpstreams/)
|
|
61
76
|
[](https://github.com/steventimes/fpstreams/blob/master/LICENSE)
|
|
77
|
+
[](https://hub.xerj.org/contributors/steventimes)
|
|
62
78
|
|
|
63
79
|
[Documentation](https://steventimes.github.io/fpstreams/) · [Browser playground](https://steventimes.github.io/fpstreams/playground/) · [Changelog](https://github.com/steventimes/fpstreams/blob/master/CHANGELOG.md) · [Contributing](https://github.com/steventimes/fpstreams/blob/master/CONTRIBUTING.md)
|
|
64
80
|
|
|
65
|
-
|
|
81
|
+
Process Python objects and async inputs with lazy pipelines, bounded concurrent
|
|
82
|
+
work, and explicit resource cleanup.
|
|
66
83
|
|
|
67
84
|
Python 3.11 or newer is required.
|
|
68
85
|
|
|
69
|
-
|
|
70
|
-
|
|
86
|
+
[Version 2.4.1](https://github.com/steventimes/fpstreams/releases/tag/v2.4.1)
|
|
87
|
+
reorganizes planner and execution internals with no public API changes.
|
|
71
88
|
Version 2.4.0 added stateful pipeline operations, composed aggregation, shared execution
|
|
72
89
|
limits, file recovery, and versioned execution metadata. See the
|
|
73
90
|
[changelog](https://github.com/steventimes/fpstreams/blob/master/CHANGELOG.md) for its scope and the
|
|
@@ -76,9 +93,21 @@ for batch and executor examples. The
|
|
|
76
93
|
[roadmap](https://steventimes.github.io/fpstreams/roadmap/#next-development-steps)
|
|
77
94
|
lists the next optimization candidates and their acceptance criteria.
|
|
78
95
|
|
|
96
|
+
Version 2.5.0 preserves custom-key hash errors in per-key reductions, Pair
|
|
97
|
+
collection, and in-memory Python joins and grouping on Python 3.14 and 3.15.
|
|
98
|
+
|
|
99
|
+
The [2.5.0 changes](CHANGELOG.md#250) include a static averaging type correction
|
|
100
|
+
with an explicit migration path. This version adds
|
|
101
|
+
[typed asynchronous folds](fpstreams/docs/reference/async_flow.md#explicit-asynchronous-accumulator-contract-250),
|
|
102
|
+
[explicit averaging operations](fpstreams/docs/reference/collecting.md#explicit-averaging-operations-250),
|
|
103
|
+
and an opt-in [strict lookup policy for null filtering](fpstreams/docs/reference/rows.md#null-filtering-and-lookup-failures-250).
|
|
104
|
+
These APIs are not in 2.4.1. `Flow.reduce` still returns `Any` to type checkers,
|
|
105
|
+
including with an annotated callback. See the [reduce guide](fpstreams/docs/reference/flow.md#reduce-types)
|
|
106
|
+
for migration examples and [contributor checks](CONTRIBUTING.md#run-the-checks) for validation.
|
|
107
|
+
|
|
79
108
|
## Install
|
|
80
109
|
|
|
81
|
-
Install
|
|
110
|
+
Install fpstreams from PyPI:
|
|
82
111
|
|
|
83
112
|
~~~bash
|
|
84
113
|
python -m pip install fpstreams
|
|
@@ -97,48 +126,47 @@ python -m pip install "fpstreams[data]" # NumPy, pandas, and PyArrow
|
|
|
97
126
|
python -m pip install "fpstreams[polars]" # Polars and PyArrow
|
|
98
127
|
~~~
|
|
99
128
|
|
|
100
|
-
##
|
|
129
|
+
## Quick start
|
|
101
130
|
|
|
102
|
-
|
|
103
|
-
Select the Python interpreter where fpstreams is installed in your editor to get
|
|
104
|
-
method completions, parameter hints, and hover documentation.
|
|
131
|
+
Pass an iterable to `flow()` and consume the result:
|
|
105
132
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
`flow([1, 2, 3])` is inferred as `Flow[int]`.
|
|
133
|
+
~~~python
|
|
134
|
+
from fpstreams import flow
|
|
109
135
|
|
|
110
|
-
|
|
136
|
+
result = flow([1, 2, 3]).filter(lambda value: value > 1).map(lambda value: value * 10).to_list()
|
|
137
|
+
assert result == [20, 30]
|
|
138
|
+
~~~
|
|
139
|
+
|
|
140
|
+
Use [async pipelines](#asynchronous-pipelines) for asynchronous callbacks and
|
|
141
|
+
[record pipelines](#record-pipelines) for grouping. The
|
|
142
|
+
[engine guide](#execution-engines) helps diagnose execution routes;
|
|
143
|
+
[resource limits](#resource-and-file-safety-controls) apply when processing larger inputs.
|
|
111
144
|
|
|
112
|
-
|
|
113
|
-
|
|
145
|
+
### Close an owned iterator
|
|
146
|
+
|
|
147
|
+
This pipeline takes three matching values and closes its generator before returning.
|
|
148
|
+
The generator prints when its `finally` block runs, so cleanup is visible:
|
|
114
149
|
|
|
115
150
|
~~~python
|
|
116
|
-
from fpstreams import
|
|
151
|
+
from fpstreams import flow
|
|
117
152
|
|
|
118
|
-
orders = [
|
|
119
|
-
{"region": "eu", "status": "paid", "amount": 24},
|
|
120
|
-
{"region": "us", "status": "paid", "amount": 20},
|
|
121
|
-
{"region": "eu", "status": "cancelled", "amount": 99},
|
|
122
|
-
{"region": "eu", "status": "paid", "amount": 24},
|
|
123
|
-
]
|
|
124
153
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
orders=agg.count(),
|
|
131
|
-
revenue=agg.sum("amount"),
|
|
132
|
-
)
|
|
133
|
-
.sort_by("region")
|
|
134
|
-
.to_list()
|
|
135
|
-
)
|
|
154
|
+
def numbers():
|
|
155
|
+
try:
|
|
156
|
+
yield from range(10)
|
|
157
|
+
finally:
|
|
158
|
+
print("source closed")
|
|
136
159
|
|
|
160
|
+
|
|
161
|
+
result = flow(numbers()).filter(lambda value: value % 2 == 0).take(3).to_list()
|
|
137
162
|
print(result)
|
|
138
|
-
#
|
|
139
|
-
#
|
|
163
|
+
# source closed
|
|
164
|
+
# [0, 2, 4]
|
|
140
165
|
~~~
|
|
141
166
|
|
|
167
|
+
See [record pipelines](#record-pipelines) for grouped aggregation, or
|
|
168
|
+
[asynchronous pipelines](#asynchronous-pipelines) for concurrent calls.
|
|
169
|
+
|
|
142
170
|
## When to use fpstreams
|
|
143
171
|
|
|
144
172
|
For a short, local transformation, a comprehension is often enough. fpstreams
|
|
@@ -174,10 +202,11 @@ standard Arrow/dataframe protocol routing, retained NumPy execution, and new
|
|
|
174
202
|
async queue, prefetch, window, and numeric terminal APIs. See the
|
|
175
203
|
[changelog](https://github.com/steventimes/fpstreams/blob/master/CHANGELOG.md) for the release summary.
|
|
176
204
|
|
|
177
|
-
|
|
178
|
-
CPython 3.14t
|
|
179
|
-
|
|
180
|
-
|
|
205
|
+
The CI matrix targets standard CPython 3.11 through 3.15.
|
|
206
|
+
Free-threaded CPython 3.14t and 3.15t are included in an
|
|
207
|
+
experimental, non-blocking CI matrix that builds the native extension and checks
|
|
208
|
+
core concurrency contracts. These builds are not release-wheel targets;
|
|
209
|
+
optional adapter compatibility is outside this matrix.
|
|
181
210
|
|
|
182
211
|
### Value pipelines
|
|
183
212
|
|
|
@@ -275,7 +304,9 @@ record column. Explicit Flow factories cover Arrow, equal-length named columns,
|
|
|
275
304
|
NumPy arrays, dataframe, Polars, typed CSV, and Parquet inputs. The `rows`
|
|
276
305
|
namespace continues to provide compatibility CSV, JSONL, SQLite, DB-API, and
|
|
277
306
|
record-oriented output methods. CSV and JSONL
|
|
278
|
-
inputs accept paths, caller-owned open handles, or replayable opener functions
|
|
307
|
+
inputs accept paths, caller-owned open handles, or replayable opener functions.
|
|
308
|
+
The following example needs `fpstreams[arrow]` and an `accounts.parquet` file
|
|
309
|
+
with `id`, `status`, and `balance` columns:
|
|
279
310
|
|
|
280
311
|
~~~python
|
|
281
312
|
from fpstreams import col, flow
|
|
@@ -324,6 +355,8 @@ asyncio.run(main())
|
|
|
324
355
|
|
|
325
356
|
Async iterators consumed by a pipeline are closed, and tasks created by the
|
|
326
357
|
pipeline are cancelled, when it finishes, errors, times out, or short-circuits.
|
|
358
|
+
Mappers must cooperate with cancellation so cleanup can finish. Offload blocking
|
|
359
|
+
work explicitly rather than running it directly inside an async mapper.
|
|
327
360
|
|
|
328
361
|
### Key/value pipelines
|
|
329
362
|
|
|
@@ -366,8 +399,13 @@ assert plan["data_movement"] == {
|
|
|
366
399
|
You can request an engine explicitly when testing parity or diagnosing a plan:
|
|
367
400
|
|
|
368
401
|
~~~python
|
|
402
|
+
from fpstreams import flow, item
|
|
403
|
+
|
|
404
|
+
pipeline = flow([1, 2, 3]).map(item * 2)
|
|
369
405
|
python_result = pipeline.with_engine("python").to_list()
|
|
370
406
|
native_result = pipeline.with_engine("native").to_list()
|
|
407
|
+
|
|
408
|
+
assert python_result == native_result == [2, 4, 6]
|
|
371
409
|
~~~
|
|
372
410
|
|
|
373
411
|
Use `run_with_report()` to get a terminal's value, recorded route, and query-owned
|
|
@@ -417,11 +455,23 @@ finish. The defaults are 100,000 rows and 64 MiB per partition, 100,000 matches
|
|
|
417
455
|
per key, 1,000,000 output rows, and three repartition levels.
|
|
418
456
|
|
|
419
457
|
~~~python
|
|
420
|
-
from fpstreams import SpillLimits, rows
|
|
458
|
+
from fpstreams import SpillLimits, agg, rows
|
|
459
|
+
|
|
460
|
+
accounts = [{"id": 1, "account_id": "a"}, {"id": 2, "account_id": "a"}]
|
|
461
|
+
balances = [{"id": 1, "balance": 10}, {"id": 2, "balance": 20}]
|
|
421
462
|
|
|
422
463
|
limits = SpillLimits(max_output_rows=250_000, max_matches_per_key=10_000)
|
|
423
|
-
|
|
424
|
-
grouped =
|
|
464
|
+
joined = rows(accounts).join(balances, on="id", partitions=32, limits=limits).to_list()
|
|
465
|
+
grouped = (
|
|
466
|
+
rows(accounts)
|
|
467
|
+
.group_by("account_id")
|
|
468
|
+
.spill(partitions=32, limits=limits)
|
|
469
|
+
.aggregate(count=agg.count())
|
|
470
|
+
.to_list()
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
assert sorted((row["id"], row["balance"]) for row in joined) == [(1, 10), (2, 20)]
|
|
474
|
+
assert grouped == [{"account_id": "a", "count": 2}]
|
|
425
475
|
~~~
|
|
426
476
|
|
|
427
477
|
Exceeding a configured partition, match, state, record, or output limit raises
|
|
@@ -446,8 +496,31 @@ only for trusted local input when an unlimited record is intentional.
|
|
|
446
496
|
once during construction and treated as one-shot; a PyArrow
|
|
447
497
|
`RecordBatchReader` is also one-shot.
|
|
448
498
|
- `flow.defer(factory)` opens a fresh source for every execution.
|
|
449
|
-
- Terminal operations
|
|
450
|
-
|
|
499
|
+
- Terminal operations release owned iterators, database cursors, and temporary
|
|
500
|
+
files, including on errors and early termination. Async execution also cancels
|
|
501
|
+
and awaits tasks it creates. File adapters distinguish owned resources from
|
|
502
|
+
caller-owned handles; see the [I/O reference](https://steventimes.github.io/fpstreams/reference/io/).
|
|
503
|
+
|
|
504
|
+
## Editor support
|
|
505
|
+
|
|
506
|
+
fpstreams includes inline type annotations, docstrings, and the `py.typed` marker.
|
|
507
|
+
Select the Python interpreter where fpstreams is installed in your editor to get
|
|
508
|
+
method completions, parameter hints, and hover documentation.
|
|
509
|
+
|
|
510
|
+
Version 2.2.1 fixes `rows` factory completion and keeps core
|
|
511
|
+
iterable element types when optional data adapters are absent. For example,
|
|
512
|
+
`flow([1, 2, 3])` is inferred as `Flow[int]`.
|
|
513
|
+
|
|
514
|
+
## Coding-agent guidance
|
|
515
|
+
|
|
516
|
+
The [Flow](fpstreams/docs/reference/flow.md#reduce-types),
|
|
517
|
+
[async Flow](fpstreams/docs/reference/async_flow.md#explicit-asynchronous-accumulator-contract-250),
|
|
518
|
+
and [Collector](fpstreams/docs/reference/collecting.md#explicit-averaging-operations-250)
|
|
519
|
+
references describe entry points, typed reductions, and migration contracts.
|
|
520
|
+
An independently supplied fpstreams usage skill must match the installed library
|
|
521
|
+
version and its API documentation. The skill is distributed separately from
|
|
522
|
+
the PyPI wheel and sdist. Neither a package install nor its presence in a
|
|
523
|
+
checkout registers it with an agent.
|
|
451
524
|
|
|
452
525
|
## v1 compatibility
|
|
453
526
|
|
|
@@ -471,25 +544,20 @@ uv sync --extra arrow --extra data --extra polars \
|
|
|
471
544
|
uv run maturin develop --release
|
|
472
545
|
uv run pytest -W error --cov=src/fpstreams --cov-branch --cov-report=term-missing
|
|
473
546
|
uv run coverage report
|
|
474
|
-
uv run ruff check src tests scripts
|
|
475
|
-
uv run ruff format --check src tests scripts
|
|
547
|
+
uv run ruff check src tests scripts
|
|
548
|
+
uv run ruff format --check src tests scripts
|
|
476
549
|
uv run mypy src/fpstreams
|
|
477
550
|
cargo test --manifest-path rust/Cargo.toml
|
|
478
551
|
uv run python scripts/build_browser_wheel.py
|
|
479
552
|
uv run mkdocs build --strict -f fpstreams/mkdocs.yml
|
|
480
553
|
~~~
|
|
481
554
|
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
filters, and optional JSON output. Competitive mode checks equivalent tasks
|
|
485
|
-
against Python, NumPy, and pandas and prints the percentage difference.
|
|
555
|
+
Performance measurements are maintained locally. Benchmark tools and raw reports
|
|
556
|
+
are not part of the public source distribution or GitHub Actions.
|
|
486
557
|
|
|
487
558
|
The source tree is organized by domain under `src/fpstreams/`: `streams`,
|
|
488
559
|
`planning`, `execution`, `collecting`, `tabular`, `expressions`, and `primitives`.
|
|
489
560
|
Small top-level modules are compatibility facades, not duplicate implementations.
|
|
490
|
-
The [Chinese code-reading guide](https://github.com/steventimes/fpstreams/blob/master/CODE_READING_GUIDE.zh-CN.md)
|
|
491
|
-
walks through the main sync, relational, spill, native, and async execution paths
|
|
492
|
-
in those domains.
|
|
493
561
|
|
|
494
562
|
Contributions are welcome. The
|
|
495
563
|
[contributing guide](https://github.com/steventimes/fpstreams/blob/master/CONTRIBUTING.md)
|