diffract-core 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {diffract_core-0.3.0 → diffract_core-0.3.1}/.gitignore +3 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/PKG-INFO +35 -10
- {diffract_core-0.3.0 → diffract_core-0.3.1}/README.md +31 -9
- {diffract_core-0.3.0 → diffract_core-0.3.1}/pyproject.toml +2 -8
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/__init__.py +3 -1
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/containers.py +4 -2
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/__init__.py +0 -6
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/aggregation.py +1 -15
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/aggregation_runner.py +33 -18
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/executor.py +20 -11
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/parameter_runner.py +18 -6
- diffract_core-0.3.1/src/diffract/core/compute/execution/restrictions.py +81 -0
- diffract_core-0.3.1/src/diffract/core/constants.py +57 -0
- diffract_core-0.3.1/src/diffract/core/data/identity.py +116 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/metadata/__init__.py +12 -0
- diffract_core-0.3.1/src/diffract/core/data/metadata/migrations.py +504 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/metadata/sqlite_index.py +37 -10
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/aggregates/metadata.py +10 -6
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/aggregates/repository.py +5 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/base.py +2 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/factory.py +20 -1
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/params/metadata.py +2 -2
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/__init__.py +9 -0
- diffract_core-0.3.1/src/diffract/session/_identifiers.py +31 -0
- diffract_core-0.3.1/src/diffract/session/errors.py +44 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/field_cache.py +10 -6
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/compute/__init__.py +121 -14
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/models/__init__.py +27 -3
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/models/parameters/__init__.py +18 -5
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/results/__init__.py +221 -66
- diffract_core-0.3.1/src/diffract/session/namespaces/results/validation.py +193 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/utils/__init__.py +30 -1
- diffract_core-0.3.1/src/diffract/session/namespaces/viz/_utils.py +33 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/viz/box.py +5 -10
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/viz/scatter.py +4 -3
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/viz/sparkline.py +6 -14
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/viz/violin.py +5 -6
- diffract_core-0.3.1/src/diffract/session/resolver.py +124 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/session.py +20 -2
- diffract_core-0.3.1/src/diffract/session/summaries.py +49 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/utils.py +18 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/data/extraction.py +23 -10
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/data/ordering.py +2 -2
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/renderer.py +3 -15
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/__init__.py +2 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/sources.py +22 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/test_executor.py +76 -10
- diffract_core-0.3.1/tests/unit/core/data/metadata/test_migrations.py +667 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/data/nn/extractors/test_numpy_extractors.py +5 -5
- diffract_core-0.3.1/tests/unit/core/data/nn/test_aggregate_uid.py +70 -0
- diffract_core-0.3.1/tests/unit/core/data/test_identity.py +98 -0
- diffract_core-0.3.1/tests/unit/session/test_compute_hints.py +181 -0
- diffract_core-0.3.1/tests/unit/session/test_export_field_validation.py +268 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/session/test_field_cache.py +17 -0
- diffract_core-0.3.1/tests/unit/session/test_identifier_validation.py +183 -0
- diffract_core-0.3.1/tests/unit/session/test_resolver.py +291 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/session/test_session.py +274 -1
- diffract_core-0.3.1/tests/unit/session/test_store_version.py +148 -0
- diffract_core-0.3.1/tests/unit/test_bench_pipeline.py +166 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/test_containers_logging.py +33 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/viz/test_common.py +60 -31
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/viz/test_plots.py +145 -0
- diffract_core-0.3.0/src/diffract/core/compute/execution/restrictions.py +0 -61
- diffract_core-0.3.0/src/diffract/core/constants.py +0 -147
- diffract_core-0.3.0/src/diffract/session/errors.py +0 -17
- diffract_core-0.3.0/src/diffract/session/namespaces/viz/_utils.py +0 -14
- diffract_core-0.3.0/tests/unit/session/test_compute_hints.py +0 -47
- {diffract_core-0.3.0 → diffract_core-0.3.1}/LICENSE +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/NOTICE +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/ruff.toml +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/configs/fast_speed_without_disk.ini +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/configs/hybrid.ini +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/configs/sqlite.ini +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/cache/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/cache/containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/cache/interface.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/cache/redis_manager.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/cache/simple_manager.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/config.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/decorator.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/exceptions.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/_types.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/enums.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/strategy.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/utils.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/extensions/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/extensions/power_law.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/extensions/rmt.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/extensions/utils.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/field_signature.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/alignment.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/heavy_tailed.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/marchenko_pastur.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/mat_decomposition.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/mat_properties.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/model_quality.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/norms.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/ranks.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/kernels/tracy_widom.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/metadata.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/registry.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/interface.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/metadata/containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/metadata/interface.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/aggregates/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/aggregates/proxy.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/aggregates/view.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/flax.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/handlers/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/handlers/base.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/handlers/flax_handlers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/handlers/numpy_handlers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/handlers/onnx_handlers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/handlers/tensorflow_handlers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/handlers/torch_handlers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/interface.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/numpy.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/onnx.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/tensorflow.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/extractors/torch.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/params/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/params/interface.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/params/proxy.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/params/repository.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/params/schema.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/nn/params/view.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/proxy.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/repository.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/utils.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/data/view.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/exporters.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/formatters/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/formatters/base.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/formatters/dict_formatter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/formatters/json_formatter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/formatters/list_formatter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/formatters/pandas_formatter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/formatters/polars_formatter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/formatters/registry.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/export/interface.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/parallel/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/parallel/containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/parallel/execution.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/parallel/runtime.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/base_manager.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/hdf5_manager.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/hybrid_manager.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/interface.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/metadata.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/ram_manager.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/serialization.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/sqlite_manager.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/storage/zarr_manager.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/utils/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/utils/build.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/utils/exceptions.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/utils/fpenv.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/utils/hashing.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/utils/imports.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/utils/math.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/py.typed +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/models/parameters/meta_patcher.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/results/eraser.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/results/injester.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/utils/merger.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/viz/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/viz/grid.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/session/namespaces/viz/heatmap.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/data/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/data/detection.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/data/filtering.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/data/provider.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/data/types.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/axis.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/coloraxis.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/configurator.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/jitter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/line.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/marker.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/overlay.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/plot.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/base/update.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/boxplot.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/cluster.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/heatmap.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/scatter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/sparkline.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/subplots/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/subplots/coloraxis.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/subplots/factory.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/subplots/grid.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/subplots/layout.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/subplots/spec.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/plots/violin.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/palettes/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/palettes/bundle.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/palettes/color.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/palettes/dashes.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/palettes/symbols.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/resolvers/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/resolvers/categorical.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/resolvers/color.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/resolvers/numeric.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/theme/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/theme/applier.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/theme/components.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/theme/presets.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/viz/styling/theme/theme.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/conftest.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/helpers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_full_stack_redis_hdf5.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_full_stack_redis_hybrid.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_full_stack_redis_sqlite.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_full_stack_simple_cache_storage.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_hybrid_zarr_s3.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_redis_cache.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_stress_concurrent_operations.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_stress_large_datasets.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/integration/test_stress_session_operations.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/conftest.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/cache/test_simple_cache.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/extensions/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/extensions/_golden/rmt_values.json +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/extensions/test_fpenv.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/extensions/test_rmt.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/conftest.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_alignment.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_heavy_tailed.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_kernel_smoke.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_marchenko_pastur.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_mat_decomposition.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_mat_properties.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_model_quality.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_norms.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_ranks.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/kernels/test_tracy_widom.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/test_default_kernels.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/test_field_signature.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/compute/test_registry.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/data/metadata/test_sqlite_index.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/data/nn/extractors/test_flax_extractors.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/data/nn/extractors/test_onnx_extractors.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/data/nn/extractors/test_tensorflow_extractors.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/data/nn/extractors/test_torch_extractors.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/data/nn/test_collection_concurrent.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/data/nn/test_parameter_module.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/export/test_exporters.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/export/test_formatter_hints.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/export/test_pandas_formatter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/export/test_polars_formatter.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/parallel/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/parallel/test_containers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/parallel/test_runtime.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/storage/test_hdf5_storage.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/storage/test_hybrid_storage.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/storage/test_ram_storage.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/storage/test_serialization.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/storage/test_sqlite_storage.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/storage/test_zarr_storage.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/utils/test_imports_utils.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/core/utils/test_session_helpers.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/session/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/session/test_results_export_format.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/test_containers_config_loading.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/viz/__init__.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/viz/conftest.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/viz/test_data_source.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/viz/test_field_ref_coercion.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/viz/test_renderer.py +0 -0
- {diffract_core-0.3.0 → diffract_core-0.3.1}/tests/unit/viz/test_themes.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: diffract-core
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Spectral analysis of neural network weights and their evolution over training
|
|
5
5
|
Project-URL: Homepage, https://github.com/Risk-AI-Research/diffract
|
|
6
6
|
Project-URL: Documentation, https://risk-ai-research.github.io/diffract/
|
|
@@ -41,6 +41,8 @@ Requires-Dist: polars>=0.20.0; extra == 'all'
|
|
|
41
41
|
Requires-Dist: pyyaml>=6.0.1; extra == 'all'
|
|
42
42
|
Requires-Dist: taichi>=1.7.3; extra == 'all'
|
|
43
43
|
Requires-Dist: torch>=2.7.0; extra == 'all'
|
|
44
|
+
Provides-Extra: bench
|
|
45
|
+
Requires-Dist: psutil>=5.9.0; extra == 'bench'
|
|
44
46
|
Provides-Extra: common
|
|
45
47
|
Requires-Dist: hydra-core>=1.3.0; extra == 'common'
|
|
46
48
|
Requires-Dist: kaleido>=0.2.1; extra == 'common'
|
|
@@ -62,6 +64,7 @@ Requires-Dist: pandas>=2.1.0; extra == 'dev'
|
|
|
62
64
|
Requires-Dist: plotly>=6.0.0; extra == 'dev'
|
|
63
65
|
Requires-Dist: polars>=0.20.0; extra == 'dev'
|
|
64
66
|
Requires-Dist: pre-commit>=3.6.0; extra == 'dev'
|
|
67
|
+
Requires-Dist: psutil>=5.9.0; extra == 'dev'
|
|
65
68
|
Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
|
|
66
69
|
Requires-Dist: pytest-mock>=3.12.0; extra == 'dev'
|
|
67
70
|
Requires-Dist: pytest-timeout>=2.4.0; extra == 'dev'
|
|
@@ -238,7 +241,9 @@ internal structure:
|
|
|
238
241
|
|
|
239
242
|
- **PARAMETER** - Operate on individual weight matrices.
|
|
240
243
|
- **IN_MODEL** - Aggregate within a single model.
|
|
241
|
-
- **CROSS_MODEL** - Compare or aggregate across models.
|
|
244
|
+
- **CROSS_MODEL** - Compare or aggregate across models. Pairwise comparisons
|
|
245
|
+
(the alignment/overlap family) compare exactly two models—scope to a pair with
|
|
246
|
+
`filter(model_ids=[a, b])`.
|
|
242
247
|
|
|
243
248
|
- **Built-in Visualization**: Publication-ready Plotly plots with theming support.
|
|
244
249
|
|
|
@@ -318,14 +323,21 @@ with attn:
|
|
|
318
323
|
Export results in various formats (`pandas`, `polars`, `dict`, `json`, or `list`):
|
|
319
324
|
|
|
320
325
|
```python
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
#
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
326
|
+
with session:
|
|
327
|
+
session.compute.apply("stable_rank", "log_norm")
|
|
328
|
+
|
|
329
|
+
# PARAMETER-level fields come from export_metrics
|
|
330
|
+
scalars_df = session.results.export_metrics("stable_rank", export_format="pandas")
|
|
331
|
+
# IN_MODEL / CROSS_MODEL fields come from export_aggregates
|
|
332
|
+
aggregates_df = session.results.export_aggregates(
|
|
333
|
+
"log_norm", export_format="pandas"
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
# Other formats work the same way
|
|
337
|
+
results = session.results.export_metrics("stable_rank", export_format="polars")
|
|
338
|
+
results = session.results.export_metrics("stable_rank", export_format="dict")
|
|
339
|
+
results = session.results.export_metrics("stable_rank", export_format="json")
|
|
340
|
+
results = session.results.export_metrics("stable_rank", export_format="list")
|
|
329
341
|
```
|
|
330
342
|
|
|
331
343
|
### Visualization
|
|
@@ -362,6 +374,19 @@ session.compute.list_available_metrics(verbose=True)
|
|
|
362
374
|
session.compute.configure_kernel("hard_rank", rtol=1e-6)
|
|
363
375
|
```
|
|
364
376
|
|
|
377
|
+
Configuration takes effect when a kernel executes. Fields already stored are reused
|
|
378
|
+
as-is — `apply` skips them by field presence, without comparing stored values to the
|
|
379
|
+
current configuration. To recompute under a new configuration, erase the stored fields
|
|
380
|
+
first:
|
|
381
|
+
|
|
382
|
+
```python
|
|
383
|
+
session.results.erase("hard_rank")
|
|
384
|
+
session.compute.apply("hard_rank") # computes with rtol=1e-6
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
See [kernel configuration](https://risk-ai-research.github.io/diffract/guide/recipes/kernels_and_compute.html)
|
|
388
|
+
for the erase-then-apply pattern with multi-output kernels and dependent fields.
|
|
389
|
+
|
|
365
390
|
### Session Management
|
|
366
391
|
|
|
367
392
|
Parameters and results stay available whenever the session is reopened. `Session()`
|
|
@@ -114,7 +114,9 @@ internal structure:
|
|
|
114
114
|
|
|
115
115
|
- **PARAMETER** - Operate on individual weight matrices.
|
|
116
116
|
- **IN_MODEL** - Aggregate within a single model.
|
|
117
|
-
- **CROSS_MODEL** - Compare or aggregate across models.
|
|
117
|
+
- **CROSS_MODEL** - Compare or aggregate across models. Pairwise comparisons
|
|
118
|
+
(the alignment/overlap family) compare exactly two models—scope to a pair with
|
|
119
|
+
`filter(model_ids=[a, b])`.
|
|
118
120
|
|
|
119
121
|
- **Built-in Visualization**: Publication-ready Plotly plots with theming support.
|
|
120
122
|
|
|
@@ -194,14 +196,21 @@ with attn:
|
|
|
194
196
|
Export results in various formats (`pandas`, `polars`, `dict`, `json`, or `list`):
|
|
195
197
|
|
|
196
198
|
```python
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
#
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
199
|
+
with session:
|
|
200
|
+
session.compute.apply("stable_rank", "log_norm")
|
|
201
|
+
|
|
202
|
+
# PARAMETER-level fields come from export_metrics
|
|
203
|
+
scalars_df = session.results.export_metrics("stable_rank", export_format="pandas")
|
|
204
|
+
# IN_MODEL / CROSS_MODEL fields come from export_aggregates
|
|
205
|
+
aggregates_df = session.results.export_aggregates(
|
|
206
|
+
"log_norm", export_format="pandas"
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
# Other formats work the same way
|
|
210
|
+
results = session.results.export_metrics("stable_rank", export_format="polars")
|
|
211
|
+
results = session.results.export_metrics("stable_rank", export_format="dict")
|
|
212
|
+
results = session.results.export_metrics("stable_rank", export_format="json")
|
|
213
|
+
results = session.results.export_metrics("stable_rank", export_format="list")
|
|
205
214
|
```
|
|
206
215
|
|
|
207
216
|
### Visualization
|
|
@@ -238,6 +247,19 @@ session.compute.list_available_metrics(verbose=True)
|
|
|
238
247
|
session.compute.configure_kernel("hard_rank", rtol=1e-6)
|
|
239
248
|
```
|
|
240
249
|
|
|
250
|
+
Configuration takes effect when a kernel executes. Fields already stored are reused
|
|
251
|
+
as-is — `apply` skips them by field presence, without comparing stored values to the
|
|
252
|
+
current configuration. To recompute under a new configuration, erase the stored fields
|
|
253
|
+
first:
|
|
254
|
+
|
|
255
|
+
```python
|
|
256
|
+
session.results.erase("hard_rank")
|
|
257
|
+
session.compute.apply("hard_rank") # computes with rtol=1e-6
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
See [kernel configuration](https://risk-ai-research.github.io/diffract/guide/recipes/kernels_and_compute.html)
|
|
261
|
+
for the erase-then-apply pattern with multi-output kernels and dependent fields.
|
|
262
|
+
|
|
241
263
|
### Session Management
|
|
242
264
|
|
|
243
265
|
Parameters and results stay available whenever the session is reopened. `Session()`
|
|
@@ -85,6 +85,7 @@ notebooks = [
|
|
|
85
85
|
"einops >=0.7.0",
|
|
86
86
|
]
|
|
87
87
|
taichi = ["taichi >=1.7.3"]
|
|
88
|
+
bench = ["psutil >=5.9.0"]
|
|
88
89
|
all = [
|
|
89
90
|
"torch >=2.7.0",
|
|
90
91
|
"plotly >=6.0.0",
|
|
@@ -115,6 +116,7 @@ dev = [
|
|
|
115
116
|
"pandas >=2.1.0",
|
|
116
117
|
"polars >=0.20.0",
|
|
117
118
|
"pre-commit >=3.6.0",
|
|
119
|
+
"psutil >=5.9.0",
|
|
118
120
|
"pytest >=7.4.0",
|
|
119
121
|
"pytest-cov >=4.0.0",
|
|
120
122
|
"pytest-mock >=3.12.0",
|
|
@@ -297,11 +299,3 @@ check-hidden = true
|
|
|
297
299
|
name = "pypi"
|
|
298
300
|
url = "https://pypi.org/simple"
|
|
299
301
|
explicit = false
|
|
300
|
-
|
|
301
|
-
[tool.commitizen]
|
|
302
|
-
name = "cz_conventional_commits"
|
|
303
|
-
tag_format = "v$version"
|
|
304
|
-
version_scheme = "pep440"
|
|
305
|
-
version_provider = "uv"
|
|
306
|
-
update_changelog_on_bump = true
|
|
307
|
-
major_version_zero = true
|
|
@@ -22,9 +22,10 @@ from __future__ import annotations
|
|
|
22
22
|
import importlib
|
|
23
23
|
from typing import TYPE_CHECKING, Any
|
|
24
24
|
|
|
25
|
-
__version__ = "0.3.
|
|
25
|
+
__version__ = "0.3.1"
|
|
26
26
|
|
|
27
27
|
from .containers import list_profiles
|
|
28
|
+
from .core.data.metadata import upgrade_metadata_index
|
|
28
29
|
from .core.data.nn.params.schema import ParameterType
|
|
29
30
|
from .session import ParameterOverrides, Session
|
|
30
31
|
|
|
@@ -37,6 +38,7 @@ __all__ = [
|
|
|
37
38
|
"Session",
|
|
38
39
|
"__version__",
|
|
39
40
|
"list_profiles",
|
|
41
|
+
"upgrade_metadata_index",
|
|
40
42
|
"viz",
|
|
41
43
|
]
|
|
42
44
|
|
|
@@ -181,8 +181,10 @@ class MainContainer(containers.DeclarativeContainer):
|
|
|
181
181
|
# Configuration
|
|
182
182
|
config = providers.Configuration()
|
|
183
183
|
|
|
184
|
-
#
|
|
185
|
-
|
|
184
|
+
# Callable, not Resource: logging is configured once at build and must stay
|
|
185
|
+
# out of the init_resources()/shutdown_resources() lifecycle, which would
|
|
186
|
+
# otherwise re-run dictConfig on every context entry and reset user handlers.
|
|
187
|
+
logging_config = providers.Callable(
|
|
186
188
|
"logging.config.dictConfig",
|
|
187
189
|
config=config.logging,
|
|
188
190
|
)
|
|
@@ -1,23 +1,17 @@
|
|
|
1
1
|
"""Core module for diffract."""
|
|
2
2
|
|
|
3
3
|
from diffract.core.constants import (
|
|
4
|
-
CONTEXT_SEPARATOR,
|
|
5
4
|
HDF5_INDEX_DATASET,
|
|
6
5
|
HDF5_INDEX_GROUP,
|
|
7
6
|
HDF5_INDEX_TOMBSTONE,
|
|
8
|
-
MODELS_CONTEXT_PREFIX,
|
|
9
|
-
PARAMS_CONTEXT_PREFIX,
|
|
10
7
|
REGEX_PREFIX,
|
|
11
8
|
WEIGHTS_FIELD,
|
|
12
9
|
)
|
|
13
10
|
|
|
14
11
|
__all__ = [
|
|
15
|
-
"CONTEXT_SEPARATOR",
|
|
16
12
|
"HDF5_INDEX_DATASET",
|
|
17
13
|
"HDF5_INDEX_GROUP",
|
|
18
14
|
"HDF5_INDEX_TOMBSTONE",
|
|
19
|
-
"MODELS_CONTEXT_PREFIX",
|
|
20
|
-
"PARAMS_CONTEXT_PREFIX",
|
|
21
15
|
"REGEX_PREFIX",
|
|
22
16
|
"WEIGHTS_FIELD",
|
|
23
17
|
]
|
{diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/aggregation.py
RENAMED
|
@@ -5,10 +5,6 @@ from __future__ import annotations
|
|
|
5
5
|
from collections.abc import Callable, Iterable
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
|
|
8
|
-
from diffract.core.constants import (
|
|
9
|
-
format_contextual_field_name,
|
|
10
|
-
format_field_suffix,
|
|
11
|
-
)
|
|
12
8
|
from diffract.core.data.nn.params.interface import IParameterView
|
|
13
9
|
|
|
14
10
|
type GroupsIterator = Iterable[tuple[str, IParameterView]]
|
|
@@ -17,7 +13,7 @@ type ContextBuilder = Callable[[str, IParameterView], AggregationContext]
|
|
|
17
13
|
|
|
18
14
|
@dataclass(frozen=True)
|
|
19
15
|
class AggregationContext:
|
|
20
|
-
"""
|
|
16
|
+
"""Structured context of an aggregated computation.
|
|
21
17
|
|
|
22
18
|
Attributes:
|
|
23
19
|
models: Model identifiers participating in aggregation.
|
|
@@ -27,16 +23,6 @@ class AggregationContext:
|
|
|
27
23
|
models: tuple[str, ...] | None = None
|
|
28
24
|
parameters: tuple[str, ...] | None = None
|
|
29
25
|
|
|
30
|
-
def to_field_suffix(self) -> str:
|
|
31
|
-
"""Build a deterministic suffix from context identifiers."""
|
|
32
|
-
return format_field_suffix(models=self.models, params=self.parameters)
|
|
33
|
-
|
|
34
|
-
def create_field_name(self, field_name: str) -> str:
|
|
35
|
-
"""Compose contextual field name using the context suffix."""
|
|
36
|
-
return format_contextual_field_name(
|
|
37
|
-
field_name, models=self.models, params=self.parameters
|
|
38
|
-
)
|
|
39
|
-
|
|
40
26
|
|
|
41
27
|
def aggregate_parameters(
|
|
42
28
|
collection: IParameterView,
|
|
@@ -52,28 +52,35 @@ class AggregationKernelRunner:
|
|
|
52
52
|
self._parallel = parallel
|
|
53
53
|
self._aggregate_repository = aggregate_repository
|
|
54
54
|
|
|
55
|
-
def run(self, kernel_name: str, parameters: IParameterView) ->
|
|
55
|
+
def run(self, kernel_name: str, parameters: IParameterView) -> set[str]:
|
|
56
56
|
"""Execute kernel with aggregation across models or parameters.
|
|
57
57
|
|
|
58
58
|
Args:
|
|
59
59
|
kernel_name: Name of the registered kernel.
|
|
60
60
|
parameters: Parameter collection to process.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
The set of field names actually written to the aggregate
|
|
64
|
+
repository (empty when every group was already computed, dropped by
|
|
65
|
+
a restriction, or there is no repository configured).
|
|
61
66
|
"""
|
|
62
67
|
pending_groups = self._collect_pending_groups(kernel_name, parameters)
|
|
63
68
|
if not pending_groups:
|
|
64
69
|
logger.debug(
|
|
65
70
|
"Skip execution of kernel '%s': no pending groups", kernel_name
|
|
66
71
|
)
|
|
67
|
-
return
|
|
72
|
+
return set()
|
|
68
73
|
|
|
69
74
|
logger.info(
|
|
70
75
|
"Executing kernel '%s' (%d groups)", kernel_name, len(pending_groups)
|
|
71
76
|
)
|
|
72
77
|
|
|
78
|
+
written: set[str] = set()
|
|
73
79
|
batches = self._batch_groups_by_budget(pending_groups, parameters)
|
|
74
80
|
for batch in batches:
|
|
75
81
|
self._prefetch_batch(batch, parameters)
|
|
76
|
-
self._execute_batch(kernel_name, batch)
|
|
82
|
+
written |= self._execute_batch(kernel_name, batch)
|
|
83
|
+
return written
|
|
77
84
|
|
|
78
85
|
def _collect_pending_groups(
|
|
79
86
|
self, kernel_name: str, parameters: IParameterView
|
|
@@ -250,8 +257,12 @@ class AggregationKernelRunner:
|
|
|
250
257
|
aggregate_uids.append(uid)
|
|
251
258
|
return aggregate_uids
|
|
252
259
|
|
|
253
|
-
def _execute_batch(self, kernel_name: str, batch: list[PendingGroup]) ->
|
|
254
|
-
"""Execute aggregation-level kernel on a batch with streaming writes.
|
|
260
|
+
def _execute_batch(self, kernel_name: str, batch: list[PendingGroup]) -> set[str]:
|
|
261
|
+
"""Execute aggregation-level kernel on a batch with streaming writes.
|
|
262
|
+
|
|
263
|
+
Returns:
|
|
264
|
+
The set of field names written for this batch.
|
|
265
|
+
"""
|
|
255
266
|
required_args = self._registry.get_fields_kernel_require(kernel_name)
|
|
256
267
|
tasks: dict[tuple[str, AggregationContext], tuple[Any, ...]] = {}
|
|
257
268
|
|
|
@@ -265,12 +276,13 @@ class AggregationKernelRunner:
|
|
|
265
276
|
self._registry.get_kernel_restrictions(kernel_name),
|
|
266
277
|
)
|
|
267
278
|
if not tasks:
|
|
268
|
-
return
|
|
279
|
+
return set()
|
|
269
280
|
|
|
270
281
|
if self._aggregate_repository is None:
|
|
271
282
|
msg = "AggregateRepository required for aggregation kernels"
|
|
272
283
|
raise RuntimeError(msg)
|
|
273
284
|
|
|
285
|
+
written: set[str] = set()
|
|
274
286
|
with self._aggregate_repository:
|
|
275
287
|
for (group_id, context), result in self._stream_results(kernel_name, tasks):
|
|
276
288
|
logger.debug(
|
|
@@ -278,7 +290,8 @@ class AggregationKernelRunner:
|
|
|
278
290
|
kernel_name,
|
|
279
291
|
group_id,
|
|
280
292
|
)
|
|
281
|
-
self._store_result(kernel_name, context, result)
|
|
293
|
+
written |= self._store_result(kernel_name, context, result)
|
|
294
|
+
return written
|
|
282
295
|
|
|
283
296
|
def _build_task_args(
|
|
284
297
|
self, required_args: tuple[str, ...], group: PendingGroup
|
|
@@ -311,35 +324,36 @@ class AggregationKernelRunner:
|
|
|
311
324
|
self, field_name: str, context: AggregationContext
|
|
312
325
|
) -> Any:
|
|
313
326
|
"""Read an aggregate value from AggregateRepository."""
|
|
314
|
-
if self._aggregate_repository is None:
|
|
315
|
-
contextual_name = context.create_field_name(field_name)
|
|
316
|
-
msg = f"AggregateRepository not configured, cannot read '{contextual_name}'"
|
|
317
|
-
raise KeyError(msg)
|
|
318
|
-
|
|
319
327
|
uid = AggregateMetadata.create_uid_from_context(
|
|
320
328
|
field_name=field_name,
|
|
321
329
|
context_models=context.models or (),
|
|
322
330
|
context_params=context.parameters or (),
|
|
323
331
|
)
|
|
324
332
|
|
|
333
|
+
if self._aggregate_repository is None:
|
|
334
|
+
msg = f"AggregateRepository not configured, cannot read '{uid}'"
|
|
335
|
+
raise KeyError(msg)
|
|
336
|
+
|
|
325
337
|
try:
|
|
326
338
|
aggregate = self._aggregate_repository.get_proxy(uid)
|
|
327
339
|
except KeyError:
|
|
328
|
-
|
|
329
|
-
msg = f"Missing contextual dependency '{contextual_name}'"
|
|
340
|
+
msg = f"Missing contextual dependency '{uid}'"
|
|
330
341
|
raise KeyError(msg) from None
|
|
331
342
|
|
|
332
343
|
if not aggregate.has_field("value"):
|
|
333
|
-
|
|
334
|
-
msg = f"Aggregate '{contextual_name}' exists but has no value"
|
|
344
|
+
msg = f"Aggregate '{uid}' exists but has no value"
|
|
335
345
|
raise KeyError(msg)
|
|
336
346
|
|
|
337
347
|
return aggregate.get_field("value")
|
|
338
348
|
|
|
339
349
|
def _store_result(
|
|
340
350
|
self, kernel_name: str, context: AggregationContext, result: Any
|
|
341
|
-
) ->
|
|
342
|
-
"""Store kernel result in aggregate repository.
|
|
351
|
+
) -> set[str]:
|
|
352
|
+
"""Store kernel result in aggregate repository.
|
|
353
|
+
|
|
354
|
+
Returns:
|
|
355
|
+
The set of field names written for this result.
|
|
356
|
+
"""
|
|
343
357
|
normalized = self._registry.normalize_kernel_result(kernel_name, result)
|
|
344
358
|
for field_name, value in normalized.items():
|
|
345
359
|
aggregate = self._aggregate_repository.get_or_create(
|
|
@@ -348,6 +362,7 @@ class AggregationKernelRunner:
|
|
|
348
362
|
context_params=context.parameters or (),
|
|
349
363
|
)
|
|
350
364
|
aggregate.set_field("value", value)
|
|
365
|
+
return set(normalized.keys())
|
|
351
366
|
|
|
352
367
|
def _stream_results(
|
|
353
368
|
self, kernel_name: str, tasks: dict[Any, tuple[Any, ...]]
|
|
@@ -89,7 +89,9 @@ class KernelExecutor:
|
|
|
89
89
|
"""Reset executed kernels set, allowing re-execution."""
|
|
90
90
|
self._executed.clear()
|
|
91
91
|
|
|
92
|
-
def execute(
|
|
92
|
+
def execute(
|
|
93
|
+
self, field_or_kernel_name: str, parameters: IParameterView
|
|
94
|
+
) -> set[str]:
|
|
93
95
|
"""Execute a kernel or produce a field with dependency resolution.
|
|
94
96
|
|
|
95
97
|
Main entry point for kernel execution. Resolves all dependencies
|
|
@@ -98,28 +100,35 @@ class KernelExecutor:
|
|
|
98
100
|
Args:
|
|
99
101
|
field_or_kernel_name: Kernel name or field name to execute/produce.
|
|
100
102
|
parameters: Parameter collection to process.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
The set of field names actually written across the resolved
|
|
106
|
+
dependency chain (empty when everything was already computed).
|
|
101
107
|
"""
|
|
102
108
|
if field_or_kernel_name in self._executed:
|
|
103
|
-
return
|
|
109
|
+
return set()
|
|
104
110
|
|
|
105
111
|
dependencies = self._registry.resolve_dependencies(field_or_kernel_name)
|
|
106
112
|
|
|
113
|
+
written: set[str] = set()
|
|
107
114
|
for dep in dependencies:
|
|
108
115
|
if dep != field_or_kernel_name and dep not in self._executed:
|
|
109
|
-
self.execute(dep, parameters)
|
|
116
|
+
written |= self.execute(dep, parameters)
|
|
110
117
|
|
|
111
118
|
if self._registry.has_kernel(field_or_kernel_name):
|
|
112
|
-
self._dispatch_kernel(field_or_kernel_name, parameters)
|
|
119
|
+
written |= self._dispatch_kernel(field_or_kernel_name, parameters)
|
|
113
120
|
self._executed.add(field_or_kernel_name)
|
|
121
|
+
return written
|
|
114
122
|
|
|
115
|
-
def _dispatch_kernel(
|
|
123
|
+
def _dispatch_kernel(
|
|
124
|
+
self, kernel_name: str, parameters: IParameterView
|
|
125
|
+
) -> set[str]:
|
|
116
126
|
"""Dispatch kernel execution to the appropriate runner."""
|
|
117
127
|
apply_level = self._registry.get_kernel_apply_level(kernel_name)
|
|
118
128
|
|
|
119
129
|
if apply_level == KernelApplyLevel.PARAMETER:
|
|
120
|
-
self._parameter_runner.run(kernel_name, parameters)
|
|
121
|
-
|
|
122
|
-
self._aggregation_runner.run(kernel_name, parameters)
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
raise RuntimeError(msg)
|
|
130
|
+
return self._parameter_runner.run(kernel_name, parameters)
|
|
131
|
+
if apply_level in (KernelApplyLevel.IN_MODEL, KernelApplyLevel.CROSS_MODEL):
|
|
132
|
+
return self._aggregation_runner.run(kernel_name, parameters)
|
|
133
|
+
msg = f"Unknown kernel apply level: {apply_level}"
|
|
134
|
+
raise RuntimeError(msg)
|
{diffract_core-0.3.0 → diffract_core-0.3.1}/src/diffract/core/compute/execution/parameter_runner.py
RENAMED
|
@@ -43,12 +43,16 @@ class ParameterKernelRunner:
|
|
|
43
43
|
self._process_pool = process_pool
|
|
44
44
|
self._parallel = parallel
|
|
45
45
|
|
|
46
|
-
def run(self, kernel_name: str, parameters: IParameterView) ->
|
|
46
|
+
def run(self, kernel_name: str, parameters: IParameterView) -> set[str]:
|
|
47
47
|
"""Execute kernel on individual parameters.
|
|
48
48
|
|
|
49
49
|
Args:
|
|
50
50
|
kernel_name: Name of the registered kernel.
|
|
51
51
|
parameters: Parameter collection to process.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
The set of field names actually written to storage (empty when
|
|
55
|
+
every parameter already had the target fields or nothing matched).
|
|
52
56
|
"""
|
|
53
57
|
target_fields = self._registry.get_fields_kernel_produce(kernel_name)
|
|
54
58
|
pending = parameters.filter_by_fields(
|
|
@@ -60,7 +64,7 @@ class ParameterKernelRunner:
|
|
|
60
64
|
logger.debug(
|
|
61
65
|
"Skip execution of kernel '%s': no pending parameters", kernel_name
|
|
62
66
|
)
|
|
63
|
-
return
|
|
67
|
+
return set()
|
|
64
68
|
|
|
65
69
|
logger.info("Executing kernel '%s'", kernel_name)
|
|
66
70
|
|
|
@@ -74,17 +78,22 @@ class ParameterKernelRunner:
|
|
|
74
78
|
)
|
|
75
79
|
)
|
|
76
80
|
if not chunks:
|
|
77
|
-
return
|
|
81
|
+
return set()
|
|
78
82
|
|
|
83
|
+
written: set[str] = set()
|
|
79
84
|
for chunk in chunks:
|
|
80
85
|
chunk.prefetch_fields(fields=required_fields, parallel=self._parallel)
|
|
81
|
-
self._execute_batch(kernel_name, chunk)
|
|
86
|
+
written |= self._execute_batch(kernel_name, chunk)
|
|
87
|
+
return written
|
|
82
88
|
|
|
83
|
-
def _execute_batch(self, kernel_name: str, batch: IParameterView) ->
|
|
89
|
+
def _execute_batch(self, kernel_name: str, batch: IParameterView) -> set[str]:
|
|
84
90
|
"""Execute parameter-level kernel on a batch with streaming writes.
|
|
85
91
|
|
|
86
92
|
Results are written immediately as they become available, reducing
|
|
87
93
|
memory pressure for heavy computations (e.g., SVD matrices).
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
The set of field names written for this batch.
|
|
88
97
|
"""
|
|
89
98
|
required_args = self._registry.get_fields_kernel_require(kernel_name)
|
|
90
99
|
tasks: dict[tuple[str, str], tuple[Any, ...]] = {}
|
|
@@ -99,16 +108,19 @@ class ParameterKernelRunner:
|
|
|
99
108
|
self._registry.get_kernel_restrictions(kernel_name),
|
|
100
109
|
)
|
|
101
110
|
if not tasks:
|
|
102
|
-
return
|
|
111
|
+
return set()
|
|
103
112
|
|
|
104
113
|
param_lookup = {(p.meta.model_id, p.meta.uid): p for p in batch}
|
|
105
114
|
|
|
115
|
+
written: set[str] = set()
|
|
106
116
|
with batch:
|
|
107
117
|
for key, result in self._stream_results(kernel_name, tasks):
|
|
108
118
|
normalized = self._registry.normalize_kernel_result(kernel_name, result)
|
|
109
119
|
param = param_lookup[key]
|
|
110
120
|
for field_name, value in normalized.items():
|
|
111
121
|
param.set_field(field_name, value)
|
|
122
|
+
written.add(field_name)
|
|
123
|
+
return written
|
|
112
124
|
|
|
113
125
|
def _stream_results(
|
|
114
126
|
self,
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Kernel restriction checking utilities."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from collections.abc import Hashable
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .aggregation import AggregationContext
|
|
10
|
+
from .enums import KernelRestrictions
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
_BINARY_MODEL_COUNT = 2
|
|
15
|
+
|
|
16
|
+
# An aggregation task id is a (group_id, AggregationContext) pair; the
|
|
17
|
+
# arity is unrelated to the binary-kernel model count.
|
|
18
|
+
_AGGREGATION_TASK_ARITY = 2
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def apply_restrictions_filter(
|
|
22
|
+
kernel_name: str,
|
|
23
|
+
tasks: dict[Hashable, tuple[Any, ...]],
|
|
24
|
+
restrictions: KernelRestrictions | None,
|
|
25
|
+
) -> None:
|
|
26
|
+
"""Drop tasks that violate kernel restrictions (mutates tasks dict).
|
|
27
|
+
|
|
28
|
+
The ``BINARY`` restriction marks a cross-model kernel that operates on a
|
|
29
|
+
model pair. A group spanning any other number of models is dropped with a
|
|
30
|
+
``WARNING`` -- never silently -- so a mis-scoped binary kernel cannot no-op
|
|
31
|
+
unnoticed (the caller pre-validates scope and raises before reaching here;
|
|
32
|
+
this is the defense-in-depth backstop).
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
kernel_name: Name of the kernel for logging.
|
|
36
|
+
tasks: Dictionary of task_id -> args to filter in place. Aggregation
|
|
37
|
+
task ids are ``(group_id, AggregationContext)``; the model count is
|
|
38
|
+
read from the context.
|
|
39
|
+
restrictions: Optional kernel restrictions to check.
|
|
40
|
+
"""
|
|
41
|
+
if not restrictions or not (restrictions & KernelRestrictions.BINARY):
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
to_remove: list[Hashable] = []
|
|
45
|
+
for task_id in tasks:
|
|
46
|
+
context = _task_context(task_id)
|
|
47
|
+
if context is None:
|
|
48
|
+
# BINARY is a cross-model restriction; there is nothing to enforce
|
|
49
|
+
# off the aggregation path.
|
|
50
|
+
continue
|
|
51
|
+
|
|
52
|
+
model_count = len(context.models or ())
|
|
53
|
+
if model_count != _BINARY_MODEL_COUNT:
|
|
54
|
+
logger.warning(
|
|
55
|
+
"Skipping kernel '%s' for group '%s': a binary cross-model kernel "
|
|
56
|
+
"requires exactly two models, but the group spans %d: %s",
|
|
57
|
+
kernel_name,
|
|
58
|
+
_task_label(task_id),
|
|
59
|
+
model_count,
|
|
60
|
+
list(context.models or ()),
|
|
61
|
+
)
|
|
62
|
+
to_remove.append(task_id)
|
|
63
|
+
|
|
64
|
+
for task_id in to_remove:
|
|
65
|
+
del tasks[task_id]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _task_context(task_id: Hashable) -> AggregationContext | None:
|
|
69
|
+
"""Extract the aggregation context from a task id, if present."""
|
|
70
|
+
if isinstance(task_id, tuple) and len(task_id) == _AGGREGATION_TASK_ARITY:
|
|
71
|
+
candidate = task_id[1]
|
|
72
|
+
if isinstance(candidate, AggregationContext):
|
|
73
|
+
return candidate
|
|
74
|
+
return None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _task_label(task_id: Hashable) -> Any:
|
|
78
|
+
"""Human-readable group label for logging."""
|
|
79
|
+
if isinstance(task_id, tuple) and task_id:
|
|
80
|
+
return task_id[0]
|
|
81
|
+
return task_id
|