mef3io 2.2.0__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/workflows/release.yml +3 -2
- {mef3io-2.2.0 → mef3io-2.3.0}/PKG-INFO +3 -2
- {mef3io-2.2.0 → mef3io-2.3.0}/README.md +2 -1
- mef3io-2.3.0/VERSION +1 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/bindings/python/mef3io_ext.cpp +5 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/CMakeLists.txt +4 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/inspect.hpp +5 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/merge.hpp +5 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/metadata.hpp +22 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/source.hpp +16 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/inspect.cpp +4 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/session_copy.cpp +194 -33
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/source.cpp +83 -1
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/validate.cpp +37 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_transform.cpp +166 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/api/python.md +4 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/index.md +1 -1
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/legacy_comparison.md +10 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/llms.txt +1 -1
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/merge.md +31 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/python.md +4 -1
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/2.2.0.md +1 -1
- mef3io-2.3.0/docs/release_notes/2.3.0.md +81 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/index.md +2 -1
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/validation.md +1 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/copy-engine.md +47 -4
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/validate-repair-recover.md +2 -2
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/warnings.md +1 -1
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/writer-precision-metadata.md +9 -6
- {mef3io-2.2.0 → mef3io-2.3.0}/mkdocs.yml +1 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/__init__.py +3 -1
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/_copy.py +45 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/_warnings.py +35 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/merge.py +9 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/transform.py +2 -0
- mef3io-2.3.0/tests/test_p53_stale_header_windows.py +126 -0
- mef3io-2.3.0/tests/test_p53_units_and_mixed_factors.py +234 -0
- mef3io-2.2.0/VERSION +0 -1
- {mef3io-2.2.0 → mef3io-2.3.0}/.gitattributes +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/ISSUE_TEMPLATE/question.yml +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/workflows/bump-version.yml +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/workflows/ci.yml +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.github/workflows/docs.yml +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/.gitignore +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/CITATION.cff +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/CLAUDE.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/CMakeLists.txt +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/CODE_OF_CONDUCT.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/CONTRIBUTING.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/CONTRIBUTORS.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/LICENSE +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/METADATA.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/SECURITY.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/README.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/_zarr_compat.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/access_modes_benchmark.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/bindings_benchmark.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/cancel_overhead_benchmark.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/compression_test.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/fs_calls_benchmark.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/long_session_benchmark.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/mef_benchmark.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/merge_one_pass_benchmark.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/benchmarks/read_many_benchmark.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/anonymize.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/byteio.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/c_api.h +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/control.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/crc.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/crypto.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/errors.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/headers.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/naming.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/parallel.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/reader.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/records.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/recover.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/red.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/session.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/session_writer.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/tar.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/time.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/transform.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/types.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/validate.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/version.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/include/mef3io/writer.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/aes.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/c_api.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/clamp.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/crc.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/durability.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/headers.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/last_segment.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/metadata.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/naming.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/naming_rules.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/no_replace.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/number_text.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/operation_marker.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/out_file.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/password.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/quantize_step.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/reader.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/records.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/recover.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/red.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/rename.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/segment_copy.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/segment_copy.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/session.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/session_layout.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/session_writer.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/sha256.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/tar.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/src/writer.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/counting_source.hpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_anonymize.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_cancel.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_concurrency.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_core.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_durability.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_inspect.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_last_segment.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_open_calls.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_parallel_open.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_password.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_read_many.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_records.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_tar.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_unicode_paths.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_write_many.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/core/tests/test_write_step.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/Doxyfile +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/anonymization.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/api/cpp/html/index.html +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/api/cpp.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/api/matlab.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/benchmarks.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/contributors.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/cpp.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/design.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/encryption_model.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/examples.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/howto/edf_to_mef.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/howto/long_recordings.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/howto/read_matlab.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/howto/read_python.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/install.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/long_recordings.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/matlab.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/mef3_format.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/metadata.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/0.2.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/0.3.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.0.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.0.1.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.1.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.1.1.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.1.2.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.1.3.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.1.4.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.2.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.3.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.4.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.5.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.5.1.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/1.6.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/2.0.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/2.0.1.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/2.1.0.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/release_notes/2.1.1.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/releasing.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/students.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/threads.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/docs/transform.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/01_write_and_read.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/02_int32_primitive.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/03_appending.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/04_segment_map.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/05_annotations.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/06_encryption.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/07_legacy_mef_tools_style.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/08_legacy_compatibility.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/09_encryption_replicability.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/10_tar_archive.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/11_legacy_rto_sessions.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/12_edf_to_mef.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/13_anonymize.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/examples/README.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/INVARIANTS.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/mef3io_handoff.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/cache.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/coverage-benchmarks-verify.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/format-gotchas.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/large-files-and-append.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/matlab-c-abi.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/module-map.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/multireader.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/naming-rename-tar.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/provenance.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/reader-placement-and-factors.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/records-and-time.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/sizing-contract.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/for_agents/notes/threads-gil-concurrency.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/+mef3io/Acquisition.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/+mef3io/Metadata.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/+mef3io/Reader.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/+mef3io/Subject.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/+mef3io/Writer.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/+mef3io/archiveSession.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/+mef3io/extractSession.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/+mef3io/recoverSession.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/README.md +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/benchmark_mef3io.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/build_mex.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/mef3io_mex.cpp +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/test_api_parity.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/matlab/test_mef3io.m +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/pyproject.toml +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/__main__.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/_archive.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/_cancel.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/_inspect.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/_multi.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/_reader.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/_writer.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/anonymize.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/cache.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/compat.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/metadata.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/naming.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/pure/__init__.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/python/mef3io/validate.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/scripts/coverage.sh +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/scripts/dev_build.sh +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/scripts/lint_matlab_tests.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/scripts/make_reader_compat_check.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/scripts/verify_local.sh +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/conftest.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/generate_golden.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/enc_both.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/enc_both.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/enc_both.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/enc_both.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/fractional_fs.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/fractional_fs.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/fractional_fs.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/fractional_fs.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/manifest.json +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/meflib_red_encrypted.mefd/l1_blocks.timd/l1_blocks-000000.segd/l1_blocks-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/meflib_red_encrypted.mefd/l1_blocks.timd/l1_blocks-000000.segd/l1_blocks-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/meflib_red_encrypted.mefd/l1_blocks.timd/l1_blocks-000000.segd/l1_blocks-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/meflib_red_encrypted.mefd/l2_blocks.timd/l2_blocks-000000.segd/l2_blocks-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/meflib_red_encrypted.mefd/l2_blocks.timd/l2_blocks-000000.segd/l2_blocks-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/meflib_red_encrypted.mefd/l2_blocks.timd/l2_blocks-000000.segd/l2_blocks-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/meflib_red_encrypted.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_channel.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_channel.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_channel.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_channel.mefd/ch2.timd/ch2-000000.segd/ch2-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_channel.mefd/ch2.timd/ch2-000000.segd/ch2-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_channel.mefd/ch2.timd/ch2-000000.segd/ch2-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_channel.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_segment.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_segment.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_segment.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_segment.mefd/ch1.timd/ch1-000001.segd/ch1-000001.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_segment.mefd/ch1.timd/ch1-000001.segd/ch1-000001.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_segment.mefd/ch1.timd/ch1-000001.segd/ch1-000001.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/multi_segment.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/nan_gap_long.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/nan_gap_long.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/nan_gap_long.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/nan_gap_long.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/nan_gap_short.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/nan_gap_short.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/nan_gap_short.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/nan_gap_short.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/plain_single.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/plain_single.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/plain_single.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/plain_single.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/with_annotations.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/with_annotations.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tidx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/with_annotations.mefd/ch1.timd/ch1-000000.segd/ch1-000000.tmet +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/with_annotations.mefd/ch1.timd/ch1.rdat +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/with_annotations.mefd/ch1.timd/ch1.ridx +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/golden/with_annotations.npz +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/make_meflib_encrypted_fixture.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/meflib_harness/red_blocks.c +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_lint_matlab_tests.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p10_metadata.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p11_tar.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p12_sizing.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p13_validate.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p14_coherence.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p15_oracle_acceptance.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p16_header_parity.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p17_large_files.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p18_recovery.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p19_block_copies.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p1_headers.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p20_segment_isolation.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p21_writer_stamp.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p22_cache_snapshot.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p22_records.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p23_segment_repaired.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p24_cancel_progress.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p24_examples.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p24_red_block_encryption.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p25_public_text.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p26_inspect.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p27_password_bytes.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p28_edf_example.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p29_anonymize.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p2_read.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p30_warning_paths.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p31_gil_release.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p32_anonymize_check.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p33_transform.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p34_merge.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p35_naming.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p36_rename.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p37_grid_ties.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p38_block_envelope.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p39_multi_reader.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p3_reader.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p40_anonymize_keep_times.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p41_write_validation.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p42_threads.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p43_write_step.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p44_segment_factors.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p45_read_many.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p46_crash_states.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p47_annotation_append.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p47_unicode_paths.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p48_write_many.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p49_encrypted_records_guard.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p4_write.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p50_copy_drop_records.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p51_merge_one_pass.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p52_merge_trimmed_part_notes.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p5_write.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p6_threads.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p7_compat_cache.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p8_append.py +0 -0
- {mef3io-2.2.0 → mef3io-2.3.0}/tests/test_p9_robustness.py +0 -0
|
@@ -137,9 +137,10 @@ jobs:
|
|
|
137
137
|
# files from the site.
|
|
138
138
|
coverage:
|
|
139
139
|
# Releases only (not branch dry runs); at -O0 with atomic counters this
|
|
140
|
-
# takes a while.
|
|
140
|
+
# takes a while: about 70 minutes at 2.2.0 (it ran out at 88% under a
|
|
141
|
+
# 60-minute limit), so the limit leaves room to grow.
|
|
141
142
|
if: startsWith(github.ref, 'refs/tags/')
|
|
142
|
-
timeout-minutes:
|
|
143
|
+
timeout-minutes: 150
|
|
143
144
|
runs-on: ubuntu-latest
|
|
144
145
|
steps:
|
|
145
146
|
- uses: actions/checkout@v4
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mef3io
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: Read, write, de-identify, cut and merge MEF 3.0 (Multiscale Electrophysiology Format) recordings: EEG, iEEG, ECG and other regularly sampled signals, from Python, MATLAB and C++
|
|
5
5
|
Keywords: MEF,MEF3,MEF 3.0,Multiscale Electrophysiology Format,EEG,iEEG,SEEG,ECoG,ECG,biosignals,electrophysiology,neuroscience,anonymization,de-identification,merge,meflib,pymef
|
|
6
6
|
Author: Filip Mivalt
|
|
@@ -418,7 +418,8 @@ pages such as
|
|
|
418
418
|
different amplifiers. One call can also trim each recording to its own
|
|
419
419
|
time range, make the result anonymous and use several threads
|
|
420
420
|
(`n_threads=`). Overlapping data is refused before anything is written, and
|
|
421
|
-
the result is checked against every source.
|
|
421
|
+
the result is checked against every source. Different units or conversion
|
|
422
|
+
factors in one channel are merged with a warning, not refused. See
|
|
422
423
|
[Merging recordings](https://bnelair.github.io/mef3io/merge/).
|
|
423
424
|
To read several recordings as one without writing anything,
|
|
424
425
|
`mef3io.MultiReader([...])` is a drop-in for `Reader` (Python).
|
|
@@ -167,7 +167,8 @@ pages such as
|
|
|
167
167
|
different amplifiers. One call can also trim each recording to its own
|
|
168
168
|
time range, make the result anonymous and use several threads
|
|
169
169
|
(`n_threads=`). Overlapping data is refused before anything is written, and
|
|
170
|
-
the result is checked against every source.
|
|
170
|
+
the result is checked against every source. Different units or conversion
|
|
171
|
+
factors in one channel are merged with a warning, not refused. See
|
|
171
172
|
[Merging recordings](https://bnelair.github.io/mef3io/merge/).
|
|
172
173
|
To read several recordings as one without writing anything,
|
|
173
174
|
`mef3io.MultiReader([...])` is a drop-in for `Reader` (Python).
|
mef3io-2.3.0/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
2.3.0
|
|
@@ -1207,6 +1207,11 @@ NB_MODULE(_mef3io, m) {
|
|
|
1207
1207
|
out["metadata_fields_changed"] = r.metadata_fields_changed;
|
|
1208
1208
|
out["sources"] = r.sources;
|
|
1209
1209
|
out["segments_split"] = r.segments_split;
|
|
1210
|
+
{
|
|
1211
|
+
nb::dict um;
|
|
1212
|
+
for (const auto& [ch, us] : r.units_mixed) um[nb::str(ch.c_str())] = us;
|
|
1213
|
+
out["units_mixed"] = um;
|
|
1214
|
+
}
|
|
1210
1215
|
out["pseudonym"] = r.pseudonym;
|
|
1211
1216
|
out["shift_us"] = opt_or_none(r.shift_us);
|
|
1212
1217
|
out["source_recording_time_offset"] = r.source_recording_time_offset;
|
|
@@ -26,6 +26,10 @@ target_link_libraries(mef3io_core PUBLIC Threads::Threads)
|
|
|
26
26
|
|
|
27
27
|
if(MSVC)
|
|
28
28
|
target_compile_options(mef3io_core PRIVATE /W4 /permissive-)
|
|
29
|
+
# Sources are UTF-8. Without /utf-8 MSVC stores a "\u00B5"-style escape in
|
|
30
|
+
# the ANSI code page (one byte 0xB5), which Python then cannot decode.
|
|
31
|
+
# PUBLIC, so the bindings and tests that use these headers compile alike.
|
|
32
|
+
target_compile_options(mef3io_core PUBLIC /utf-8)
|
|
29
33
|
else()
|
|
30
34
|
target_compile_options(mef3io_core PRIVATE -Wall -Wextra)
|
|
31
35
|
endif()
|
|
@@ -92,6 +92,11 @@ struct InspectChannel {
|
|
|
92
92
|
/// (#150; each segment's is in InspectSegment::units_conversion_factor).
|
|
93
93
|
std::optional<sf8> sampling_frequency, units_conversion_factor;
|
|
94
94
|
std::optional<std::string> units_description;
|
|
95
|
+
/// Every DISTINCT units string the readable segments declare, in order of
|
|
96
|
+
/// first appearance (as stored; "uV", "\u00B5V" and "\u03BCV" count as one).
|
|
97
|
+
/// More than one = the channel's segments disagree (validator `units.mixed`).
|
|
98
|
+
/// C++ only: not in the Python summary.
|
|
99
|
+
std::vector<std::string> units_all;
|
|
95
100
|
/// Section 2's free-text channel description, as the first segment whose
|
|
96
101
|
/// section 2 is readable states it (segments can differ; the identity
|
|
97
102
|
/// table lists every distinct value). An IDENTITY value, so it is returned
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
#include <atomic>
|
|
15
15
|
#include <cstddef>
|
|
16
|
+
#include <map>
|
|
16
17
|
#include <optional>
|
|
17
18
|
#include <string>
|
|
18
19
|
#include <utility>
|
|
@@ -84,6 +85,10 @@ struct MergeReport : TransformReport {
|
|
|
84
85
|
si8 source_recording_time_offset = 0;
|
|
85
86
|
si8 record_text_fields_changed = 0;
|
|
86
87
|
si8 record_bodies_cleared = 0;
|
|
88
|
+
/// Channels whose KEPT segments carry different units strings (free text,
|
|
89
|
+
/// never a refusal): channel -> the distinct strings as stored, in order.
|
|
90
|
+
/// Segments keep their own strings; the channel label is the first kept one's.
|
|
91
|
+
std::map<std::string, std::vector<std::string>> units_mixed;
|
|
87
92
|
/// anonymize only. THE RE-IDENTIFICATION KEY over all sources (earliest
|
|
88
93
|
/// start, latest end, and per kept channel): in memory only, as in
|
|
89
94
|
/// AnonymizeReport.
|
|
@@ -62,4 +62,26 @@ TimeSeriesMetadata load_time_series_metadata(std::span<const ui1> tmet_bytes,
|
|
|
62
62
|
TimeSeriesMetadata load_time_series_metadata_partial(std::span<const ui1> tmet_bytes,
|
|
63
63
|
const std::string& password = "");
|
|
64
64
|
|
|
65
|
+
/// Units strings are free text. Two of them are the same unit when they are
|
|
66
|
+
/// equal after mapping the micro prefix (U+00B5 MICRO SIGN, U+03BC GREEK SMALL
|
|
67
|
+
/// LETTER MU, as UTF-8) to ASCII "u": "uV", "\u00B5V" and "\u03BCV" agree.
|
|
68
|
+
/// Nothing else is folded. Used only to decide whether units MIX.
|
|
69
|
+
inline std::string units_comparable(const std::string& s) {
|
|
70
|
+
std::string out;
|
|
71
|
+
out.reserve(s.size());
|
|
72
|
+
for (std::size_t i = 0; i < s.size(); ++i) {
|
|
73
|
+
const unsigned char c = static_cast<unsigned char>(s[i]);
|
|
74
|
+
if (i + 1 < s.size()) {
|
|
75
|
+
const unsigned char d = static_cast<unsigned char>(s[i + 1]);
|
|
76
|
+
if ((c == 0xC2 && d == 0xB5) || (c == 0xCE && d == 0xBC)) {
|
|
77
|
+
out += 'u';
|
|
78
|
+
++i;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
out += s[i];
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
|
|
65
87
|
} // namespace mef3io
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
/// uncompressed .mefd.tar archive (random access into member byte ranges).
|
|
4
4
|
#pragma once
|
|
5
5
|
|
|
6
|
+
#include <atomic>
|
|
7
|
+
#include <cstddef>
|
|
6
8
|
#include <cstdint>
|
|
7
9
|
#include <filesystem>
|
|
8
10
|
#include <memory>
|
|
@@ -82,6 +84,8 @@ class DirectorySource : public SessionSource {
|
|
|
82
84
|
|
|
83
85
|
private:
|
|
84
86
|
std::string join(const std::string& rel) const;
|
|
87
|
+
std::shared_ptr<const std::vector<ui1>> memo_get(const std::string& path, const std::string& rel,
|
|
88
|
+
bool fill) const;
|
|
85
89
|
std::string root_;
|
|
86
90
|
std::shared_ptr<detail::HandleCache> cache_; // null = plain reads
|
|
87
91
|
};
|
|
@@ -136,6 +140,18 @@ class ReadHandleScope {
|
|
|
136
140
|
/// opened at most N times in a merge".
|
|
137
141
|
inline void (*source_open_observer)(const std::string& path) = nullptr;
|
|
138
142
|
|
|
143
|
+
/// TEST HOOK: called with the path of every positional read a DirectorySource
|
|
144
|
+
/// really issues (a request on a network drive; memo hits do not count).
|
|
145
|
+
inline void (*source_read_observer)(const std::string& path) = nullptr;
|
|
146
|
+
|
|
147
|
+
/// TEST HOOK: total bytes of positional reads a DirectorySource really issued
|
|
148
|
+
/// (memo hits do not count). Tests reset it to 0 and compare.
|
|
149
|
+
inline std::atomic<std::uint64_t> source_read_bytes{0};
|
|
150
|
+
|
|
151
|
+
/// Budget of the small-file memo (a HandleCache holds at most this many bytes
|
|
152
|
+
/// of .tmet/.tidx/.ridx contents). A variable only so a test can lower it.
|
|
153
|
+
inline std::size_t memo_total_budget = 256u << 20;
|
|
154
|
+
|
|
139
155
|
/// TEST HOOK: called before each positional read through a cached handle;
|
|
140
156
|
/// a non-zero return is raised as that errno (POSIX) in place of the read.
|
|
141
157
|
inline int (*read_fault_hook)(const std::string& path) = nullptr;
|
|
@@ -478,6 +478,10 @@ SessionSummary detail::inspect_session_through(std::shared_ptr<const SessionSour
|
|
|
478
478
|
md.section2.units_conversion_factor)) {
|
|
479
479
|
factor_differs = true; // #150: segments store different factors
|
|
480
480
|
}
|
|
481
|
+
if (std::none_of(ic.units_all.begin(), ic.units_all.end(), [&](const std::string& u) {
|
|
482
|
+
return units_comparable(u) == units_comparable(md.section2.units_description);
|
|
483
|
+
}))
|
|
484
|
+
ic.units_all.push_back(md.section2.units_description);
|
|
481
485
|
// File-derived: a negative count is nonsense, and the sum must not overflow.
|
|
482
486
|
if (md.section2.number_of_samples < 0)
|
|
483
487
|
n_samples_known = false;
|
|
@@ -1079,6 +1079,8 @@ si8 longest_block_us(const fmt::TimeSeriesMetadataSection2& m) {
|
|
|
1079
1079
|
return static_cast<si8>(std::llround(n * 1e6 / m.sampling_frequency)) + 1'000'000;
|
|
1080
1080
|
}
|
|
1081
1081
|
|
|
1082
|
+
constexpr std::size_t WHOLE_INDEX_BYTES = 512 * 1024;
|
|
1083
|
+
|
|
1082
1084
|
/// The index entries [first, last) of a segment a cut [t0, t1) must look at,
|
|
1083
1085
|
/// found by binary search on the index's start times (O(log n) small reads:
|
|
1084
1086
|
/// never the whole index, never the data): from two blocks before the first
|
|
@@ -1095,7 +1097,15 @@ std::pair<std::size_t, std::size_t> window_entries(const SessionSource& src,
|
|
|
1095
1097
|
if (size < fmt::UNIVERSAL_HEADER_BYTES) return {0, 0};
|
|
1096
1098
|
const std::size_t n =
|
|
1097
1099
|
static_cast<std::size_t>((size - fmt::UNIVERSAL_HEADER_BYTES) / fmt::TIME_SERIES_INDEX_BYTES);
|
|
1100
|
+
// A small index (<= WHOLE_INDEX_BYTES, ~9k blocks) is read in ONE request
|
|
1101
|
+
// and searched in memory: on a network drive each of the ~15 probes of a
|
|
1102
|
+
// binary search is a round trip, dearer than the few KB they save.
|
|
1103
|
+
std::vector<ui1> whole;
|
|
1104
|
+
if (n * fmt::TIME_SERIES_INDEX_BYTES <= WHOLE_INDEX_BYTES)
|
|
1105
|
+
whole = src.read_range(tidx_rel, fmt::UNIVERSAL_HEADER_BYTES, n * fmt::TIME_SERIES_INDEX_BYTES);
|
|
1098
1106
|
auto time_at = [&](std::size_t i) {
|
|
1107
|
+
if (!whole.empty())
|
|
1108
|
+
return to_user_time(read<si8>(whole, i * fmt::TIME_SERIES_INDEX_BYTES + 8), rto);
|
|
1099
1109
|
const auto b = src.read_range(
|
|
1100
1110
|
tidx_rel, fmt::UNIVERSAL_HEADER_BYTES + i * fmt::TIME_SERIES_INDEX_BYTES + 8, 8);
|
|
1101
1111
|
return to_user_time(read<si8>(b, 0), rto);
|
|
@@ -1115,6 +1125,91 @@ std::pair<std::size_t, std::size_t> window_entries(const SessionSource& src,
|
|
|
1115
1125
|
return {a >= 2 ? a - 2 : 0, std::min(n, b + 1)};
|
|
1116
1126
|
}
|
|
1117
1127
|
|
|
1128
|
+
/// The span of ALL of a segment's index entries, read once (one range read of
|
|
1129
|
+
/// the entries, served by the per-copy handle cache), by the reader's own
|
|
1130
|
+
/// rule (Session::learn_span, core/src/session.cpp:131-160): every entry's
|
|
1131
|
+
/// [start, start + llround(n * 1e6 / fs)); an entry with no usable count
|
|
1132
|
+
/// (0 = damaged, NO_ENTRY) or an unusable rate leaves the span open (the
|
|
1133
|
+
/// reader's span_open: such a segment is never skipped).
|
|
1134
|
+
struct IndexSpan {
|
|
1135
|
+
bool have = false; // at least one entry was read
|
|
1136
|
+
bool open = false;
|
|
1137
|
+
si8 lo = 0, hi = 0;
|
|
1138
|
+
si8 first_start = 0; // the first entry's start, and the last entry's end
|
|
1139
|
+
si8 last_end = 0; // (start + n / fs; the start when fs is unusable)
|
|
1140
|
+
};
|
|
1141
|
+
|
|
1142
|
+
IndexSpan index_span(const SessionSource& src, const std::string& tidx_rel, si8 rto, sf8 fs) {
|
|
1143
|
+
IndexSpan sp;
|
|
1144
|
+
const std::uint64_t isize = src.file_size(tidx_rel);
|
|
1145
|
+
if (isize < fmt::UNIVERSAL_HEADER_BYTES + fmt::TIME_SERIES_INDEX_BYTES) return sp;
|
|
1146
|
+
const std::size_t cnt = static_cast<std::size_t>((isize - fmt::UNIVERSAL_HEADER_BYTES) /
|
|
1147
|
+
fmt::TIME_SERIES_INDEX_BYTES);
|
|
1148
|
+
const auto es = src.read_range(tidx_rel, fmt::UNIVERSAL_HEADER_BYTES,
|
|
1149
|
+
cnt * fmt::TIME_SERIES_INDEX_BYTES);
|
|
1150
|
+
sp.have = true;
|
|
1151
|
+
sp.first_start = to_user_time(read<si8>(es, 8), rto);
|
|
1152
|
+
{
|
|
1153
|
+
const std::size_t o = (cnt - 1) * fmt::TIME_SERIES_INDEX_BYTES;
|
|
1154
|
+
const si8 t = to_user_time(read<si8>(es, o + 8), rto);
|
|
1155
|
+
sp.last_end = fs > 0 ? sample_time(t, read<ui4>(es, o + 24), fs) : t;
|
|
1156
|
+
}
|
|
1157
|
+
sp.open = !(fs > 0) || !std::isfinite(fs);
|
|
1158
|
+
sp.lo = std::numeric_limits<si8>::max();
|
|
1159
|
+
sp.hi = std::numeric_limits<si8>::min();
|
|
1160
|
+
for (std::size_t i = 0; i < cnt && !sp.open; ++i) {
|
|
1161
|
+
const ui4 n = read<ui4>(es, i * fmt::TIME_SERIES_INDEX_BYTES + 24);
|
|
1162
|
+
if (n == 0 || n == fmt::UI4_NO_ENTRY) {
|
|
1163
|
+
sp.open = true;
|
|
1164
|
+
break;
|
|
1165
|
+
}
|
|
1166
|
+
const si8 b0 = to_user_time(read<si8>(es, i * fmt::TIME_SERIES_INDEX_BYTES + 8), rto);
|
|
1167
|
+
sp.lo = std::min(sp.lo, b0);
|
|
1168
|
+
sp.hi = std::max(sp.hi, b0 + static_cast<si8>(std::llround(static_cast<sf8>(n) * 1e6 / fs)));
|
|
1169
|
+
}
|
|
1170
|
+
return sp;
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
/// A segment's span in user uUTC, for deciding whether a cut [t0, t1) can
|
|
1174
|
+
/// touch it: the header's start/end WIDENED to the span of the index (all its
|
|
1175
|
+
/// entries, index_span). NOT padded: the reader skips by exactly this
|
|
1176
|
+
/// span, and a pad would keep segments the window has no samples in (the
|
|
1177
|
+
/// copy would then hold empty segments). A header time is
|
|
1178
|
+
/// stale or late in the field (an interrupted append, a short-ending writer;
|
|
1179
|
+
/// #146 lost samples on both, and #184 dropped a merge source's whole window
|
|
1180
|
+
/// on a stale end), so it alone never rules a segment out - the blocks do, as
|
|
1181
|
+
/// the reader does. An unset header side, or an open index span, is unbounded.
|
|
1182
|
+
std::pair<si8, si8> widened_span(const IndexSpan& ix, si8 hdr_start, si8 hdr_end, si8 rto) {
|
|
1183
|
+
constexpr si8 LO = std::numeric_limits<si8>::min(), HI = std::numeric_limits<si8>::max();
|
|
1184
|
+
si8 a = is_sentinel(hdr_start, false) ? LO : to_user_time(hdr_start, rto);
|
|
1185
|
+
si8 b = is_sentinel(hdr_end, false) ? HI : to_user_time(hdr_end, rto);
|
|
1186
|
+
if (ix.have) {
|
|
1187
|
+
if (ix.open) return {LO, HI};
|
|
1188
|
+
a = std::min(a, ix.lo);
|
|
1189
|
+
b = std::max(b, ix.hi);
|
|
1190
|
+
}
|
|
1191
|
+
return {a, b};
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
std::pair<si8, si8> segment_span(const SessionSource& src, const std::string& tidx_rel, si8 rto,
|
|
1195
|
+
sf8 fs, si8 hdr_start, si8 hdr_end) {
|
|
1196
|
+
return widened_span(index_span(src, tidx_rel, rto, fs), hdr_start, hdr_end, rto);
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
/// Whether the span [a, b) lies wholly outside [t0 - margin, t1 + margin).
|
|
1200
|
+
bool span_outside(const std::pair<si8, si8>& span, si8 hdr_start, si8 hdr_end, si8 t0, si8 t1,
|
|
1201
|
+
si8 margin = 0) {
|
|
1202
|
+
if (is_sentinel(hdr_start, false) || is_sentinel(hdr_end, false)) return false;
|
|
1203
|
+
return !(span.first < saturating_add(t1, margin) && span.second > saturating_add(t0, -margin));
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
bool segment_outside(const SessionSource& src, const std::string& tidx_rel, si8 rto, sf8 fs,
|
|
1207
|
+
si8 hdr_start, si8 hdr_end, si8 t0, si8 t1, si8 margin = 0) {
|
|
1208
|
+
if (is_sentinel(hdr_start, false) || is_sentinel(hdr_end, false)) return false;
|
|
1209
|
+
return span_outside(segment_span(src, tidx_rel, rto, fs, hdr_start, hdr_end),
|
|
1210
|
+
hdr_start, hdr_end, t0, t1, margin);
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1118
1213
|
/// A cut's user time stored the way `like` was stored (meflib's negated
|
|
1119
1214
|
/// form, or absolute); an unset `like` gets the negated form.
|
|
1120
1215
|
si8 stored_like(si8 like, si8 user, si8 rto) {
|
|
@@ -1410,9 +1505,9 @@ class Copier {
|
|
|
1410
1505
|
}
|
|
1411
1506
|
if (window_) {
|
|
1412
1507
|
// A segment outside the window is not read at all.
|
|
1413
|
-
|
|
1414
|
-
if (
|
|
1415
|
-
|
|
1508
|
+
// (judged by the blocks too, not the header alone: #184)
|
|
1509
|
+
if (segment_outside(src_, seg.tidx, t.rto, seg_fs_, read<si8>(t.bytes, UH_START),
|
|
1510
|
+
read<si8>(t.bytes, UH_END), window_->first, window_->second))
|
|
1416
1511
|
return false;
|
|
1417
1512
|
}
|
|
1418
1513
|
if (reshape) {
|
|
@@ -3346,10 +3441,11 @@ VerifyReport verify_impl(const std::string& src_path, const SessionSource& src,
|
|
|
3346
3441
|
std::vector<std::pair<si8, si8>> spans; // source segments, user uUTC
|
|
3347
3442
|
if (cut)
|
|
3348
3443
|
for (const auto& x : sc.segments) {
|
|
3349
|
-
const
|
|
3350
|
-
const
|
|
3351
|
-
|
|
3352
|
-
|
|
3444
|
+
const TmetImage t = open_tmet(src, x.tmet, o.password);
|
|
3445
|
+
const auto m = fmt::TimeSeriesMetadataSection2::parse(
|
|
3446
|
+
std::span<const ui1>(t.bytes).subspan(S2, S2_BYTES));
|
|
3447
|
+
spans.push_back(segment_span(src, x.tidx, t.rto, m.sampling_frequency,
|
|
3448
|
+
read<si8>(t.bytes, UH_START), read<si8>(t.bytes, UH_END)));
|
|
3353
3449
|
}
|
|
3354
3450
|
std::size_t next = 0;
|
|
3355
3451
|
int expected_number = 0; // a cut numbers each channel's segments from 0
|
|
@@ -3738,26 +3834,29 @@ Preflight preflight(const std::string& src_path, const std::string& dst_path, co
|
|
|
3738
3834
|
for (const auto& c : plan.layout.channels) {
|
|
3739
3835
|
if (!pf.kept.count(c.name)) continue;
|
|
3740
3836
|
bool has = false;
|
|
3741
|
-
|
|
3742
|
-
|
|
3837
|
+
// Two passes, same answer as one over the day margin, far fewer
|
|
3838
|
+
// requests: first the segments whose header span reaches the window
|
|
3839
|
+
// (the usual place its samples are); only a channel still without any
|
|
3840
|
+
// looks at the rest of the margin. The margin exists because a header
|
|
3841
|
+
// time can be stale (an appended segment's END, a late START: #146),
|
|
3842
|
+
// so a segment just outside the window by its header may still hold
|
|
3843
|
+
// samples of it; it matters only for a channel nothing else answers.
|
|
3844
|
+
struct Head {
|
|
3845
|
+
bool exact = true; // the header span overlaps the window (or has no times)
|
|
3846
|
+
bool margin = true; // ... or is within a day of it
|
|
3847
|
+
bool done = false;
|
|
3848
|
+
};
|
|
3849
|
+
std::vector<Head> heads(c.segments.size());
|
|
3850
|
+
auto examine = [&](const auto& seg) {
|
|
3743
3851
|
ctl.check();
|
|
3744
3852
|
try {
|
|
3745
|
-
const auto uh = src.read_range(seg.tmet, 0, UH);
|
|
3746
|
-
const si8 a = read<si8>(uh, UH_START), b = read<si8>(uh, UH_END);
|
|
3747
|
-
// A first look with the segment's own offset (a day's margin;
|
|
3748
|
-
// the session's may differ from it when segments disagree).
|
|
3749
|
-
const si8 r = segment_rto_of(plan, seg.tmet);
|
|
3750
|
-
if (!is_sentinel(a, false) && !is_sentinel(b, false) &&
|
|
3751
|
-
!(to_user_time(a, r) < saturating_add(t1, DAY_US) &&
|
|
3752
|
-
to_user_time(b, r) > saturating_add(t0, -DAY_US)))
|
|
3753
|
-
continue;
|
|
3754
3853
|
const TmetImage t = open_tmet(src, seg.tmet, o.password);
|
|
3755
3854
|
const auto m = fmt::TimeSeriesMetadataSection2::parse(
|
|
3756
3855
|
std::span<const ui1>(t.bytes).subspan(S2, S2_BYTES));
|
|
3757
3856
|
const sf8 fs = m.sampling_frequency;
|
|
3758
|
-
if (!(fs > 0))
|
|
3857
|
+
if (!(fs > 0)) return;
|
|
3759
3858
|
const auto [f, l] = window_entries(src, seg.tidx, t.rto, t0, t1, longest_block_us(m));
|
|
3760
|
-
if (f >= l)
|
|
3859
|
+
if (f >= l) return;
|
|
3761
3860
|
const auto es = src.read_range(seg.tidx, UH + f * ENTRY, (l - f) * ENTRY);
|
|
3762
3861
|
for (std::size_t i = 0; i < l - f && !has; ++i) {
|
|
3763
3862
|
const auto [i0, i1] =
|
|
@@ -3769,7 +3868,40 @@ Preflight preflight(const std::string& src_path, const std::string& dst_path, co
|
|
|
3769
3868
|
add_exception(std::current_exception(), "source-damaged", "source-crc",
|
|
3770
3869
|
"source-unreadable", c.name);
|
|
3771
3870
|
}
|
|
3871
|
+
};
|
|
3872
|
+
for (std::size_t k = 0; k < c.segments.size() && !has; ++k) {
|
|
3873
|
+
const auto& seg = c.segments[k];
|
|
3874
|
+
ctl.check();
|
|
3875
|
+
try {
|
|
3876
|
+
// The span is the header's WIDENED to the segment's index
|
|
3877
|
+
// (segment_span, as the reader's learn_span; #184), never the
|
|
3878
|
+
// header alone: a stale end or late start still reaches the
|
|
3879
|
+
// window by its blocks. The segment's own offset (the session's
|
|
3880
|
+
// may differ from it when segments disagree).
|
|
3881
|
+
const TmetImage t = open_tmet(src, seg.tmet, o.password);
|
|
3882
|
+
const sf8 sfs = fmt::TimeSeriesMetadataSection2::parse(
|
|
3883
|
+
std::span<const ui1>(t.bytes).subspan(S2, S2_BYTES))
|
|
3884
|
+
.sampling_frequency;
|
|
3885
|
+
const si8 a = read<si8>(t.bytes, UH_START), b = read<si8>(t.bytes, UH_END);
|
|
3886
|
+
if (!is_sentinel(a, false) && !is_sentinel(b, false)) {
|
|
3887
|
+
const auto sp = segment_span(src, seg.tidx, segment_rto_of(plan, seg.tmet), sfs, a, b);
|
|
3888
|
+
heads[k].exact = sp.first < t1 && sp.second > t0;
|
|
3889
|
+
heads[k].margin = sp.first < saturating_add(t1, DAY_US) &&
|
|
3890
|
+
sp.second > saturating_add(t0, -DAY_US);
|
|
3891
|
+
}
|
|
3892
|
+
} catch (...) {
|
|
3893
|
+
add_exception(std::current_exception(), "source-damaged", "source-crc",
|
|
3894
|
+
"source-unreadable", c.name);
|
|
3895
|
+
heads[k].exact = heads[k].margin = false;
|
|
3896
|
+
continue;
|
|
3897
|
+
}
|
|
3898
|
+
if (heads[k].exact) {
|
|
3899
|
+
heads[k].done = true;
|
|
3900
|
+
examine(seg);
|
|
3901
|
+
}
|
|
3772
3902
|
}
|
|
3903
|
+
for (std::size_t k = 0; k < c.segments.size() && !has; ++k)
|
|
3904
|
+
if (!heads[k].done && heads[k].margin) examine(c.segments[k]);
|
|
3773
3905
|
if (has) any = true;
|
|
3774
3906
|
else pf.without_samples.push_back(c.name);
|
|
3775
3907
|
}
|
|
@@ -4542,6 +4674,7 @@ struct MergePiece {
|
|
|
4542
4674
|
si8 start = 0; ///< its first block's time (index, user uUTC)
|
|
4543
4675
|
si8 lo = 0, hi = 0; ///< the record times its segment-level annotations take
|
|
4544
4676
|
Uuid level{}; ///< the result segment's level UUID (all its files)
|
|
4677
|
+
std::string units; ///< its source segment's units string, as stored
|
|
4545
4678
|
};
|
|
4546
4679
|
|
|
4547
4680
|
struct MergeSourcePlan {
|
|
@@ -4561,6 +4694,9 @@ struct MergePlan {
|
|
|
4561
4694
|
/// Kept channels left out because no source has a sample of them in its
|
|
4562
4695
|
/// range (also in `excluded`).
|
|
4563
4696
|
std::vector<std::string> without_samples;
|
|
4697
|
+
/// Channels whose kept segments carry different units strings (free text,
|
|
4698
|
+
/// never a refusal): the validator's `units.mixed` on them is expected.
|
|
4699
|
+
std::map<std::string, std::vector<std::string>> units_mixed;
|
|
4564
4700
|
bool anonymize = false;
|
|
4565
4701
|
std::optional<si8> shift; // anonymize: the shift (unset = times kept)
|
|
4566
4702
|
std::string pseudonym; // anonymize
|
|
@@ -4989,6 +5125,7 @@ MergePlan preflight_merge(const std::vector<std::string>& sources, const std::st
|
|
|
4989
5125
|
bool read = false;
|
|
4990
5126
|
};
|
|
4991
5127
|
std::vector<Seg> segs;
|
|
5128
|
+
std::map<const detail::LayoutSegment*, std::string> seg_units;
|
|
4992
5129
|
for (std::size_t k = 0; k < m.sources.size(); ++k) {
|
|
4993
5130
|
const auto& sp = *m.sources[k];
|
|
4994
5131
|
if (!sp.used || !sp.pf.kept.count(ch)) continue;
|
|
@@ -5008,28 +5145,26 @@ MergePlan preflight_merge(const std::vector<std::string>& sources, const std::st
|
|
|
5008
5145
|
// stale, and two segments are compared only where spans meet.
|
|
5009
5146
|
si8 a = std::numeric_limits<si8>::max(), b = std::numeric_limits<si8>::min();
|
|
5010
5147
|
const si8 ua = read<si8>(t.bytes, UH_START), ub = read<si8>(t.bytes, UH_END);
|
|
5011
|
-
//
|
|
5012
|
-
//
|
|
5013
|
-
|
|
5014
|
-
|
|
5148
|
+
// One read of the index (index_span); a cut copies no segment
|
|
5149
|
+
// outside its range, judged by the blocks too, not the header
|
|
5150
|
+
// alone (the copy's own rule, #184).
|
|
5151
|
+
const IndexSpan ix = index_span(src, seg.tidx, t.rto, fs);
|
|
5152
|
+
if (win && span_outside(widened_span(ix, ua, ub, t.rto), ua, ub,
|
|
5153
|
+
win->first, win->second))
|
|
5015
5154
|
continue;
|
|
5016
5155
|
if (win && !(fs > 0)) continue; // the source's own pre-flight left it out
|
|
5017
5156
|
if (!is_sentinel(ua, false)) a = to_user_time(ua, t.rto);
|
|
5018
5157
|
if (!is_sentinel(ub, false)) b = to_user_time(ub, t.rto);
|
|
5019
|
-
|
|
5020
|
-
|
|
5021
|
-
|
|
5022
|
-
const auto e0 = src.read_range(seg.tidx, UH, ENTRY);
|
|
5023
|
-
const auto el = src.read_range(seg.tidx, UH + (cnt - 1) * ENTRY, ENTRY);
|
|
5024
|
-
a = std::min(a, to_user_time(read<si8>(e0, 8), t.rto));
|
|
5025
|
-
b = std::max(b, sample_time(to_user_time(read<si8>(el, 8), t.rto),
|
|
5026
|
-
read<ui4>(el, 24), fs));
|
|
5158
|
+
if (ix.have) {
|
|
5159
|
+
a = std::min(a, ix.first_start);
|
|
5160
|
+
if (fs > 0) b = std::max(b, ix.last_end);
|
|
5027
5161
|
}
|
|
5028
5162
|
// (Not clipped to the range: kept blocks extend past its end, by a
|
|
5029
5163
|
// sample when exact, by a block otherwise. The span only prunes
|
|
5030
5164
|
// which segments are compared; the comparison itself is on the
|
|
5031
5165
|
// blocks the copy keeps, index_blocks.)
|
|
5032
5166
|
segs.push_back({k, &seg, t.rto, fs, longest_block_us(s2), a, b, {}, false});
|
|
5167
|
+
seg_units[&seg] = s2.units_description;
|
|
5033
5168
|
} catch (...) {
|
|
5034
5169
|
// the source's own pre-flight already reported it
|
|
5035
5170
|
}
|
|
@@ -5128,6 +5263,30 @@ MergePlan preflight_merge(const std::vector<std::string>& sources, const std::st
|
|
|
5128
5263
|
}
|
|
5129
5264
|
pc.level = random_uuid();
|
|
5130
5265
|
}
|
|
5266
|
+
// Units are free text and never block a merge. Decided on the segments the
|
|
5267
|
+
// result actually KEEPS (after the cut): when they carry different strings
|
|
5268
|
+
// (after folding the micro prefix: "uV" = "\u00B5V" = "\u03BCV") the channel is
|
|
5269
|
+
// listed in units_mixed and noted. Segments keep their own strings and
|
|
5270
|
+
// samples are never rescaled; the channel's label is the first kept
|
|
5271
|
+
// segment's.
|
|
5272
|
+
if (!pieces.empty()) {
|
|
5273
|
+
for (auto& pc : pieces) pc.units = seg_units[pc.segment];
|
|
5274
|
+
std::vector<std::string> distinct;
|
|
5275
|
+
for (const auto& pc : pieces)
|
|
5276
|
+
if (std::none_of(distinct.begin(), distinct.end(), [&](const std::string& u) {
|
|
5277
|
+
return units_comparable(u) == units_comparable(pc.units);
|
|
5278
|
+
}))
|
|
5279
|
+
distinct.push_back(pc.units);
|
|
5280
|
+
if (distinct.size() > 1) {
|
|
5281
|
+
std::string listed;
|
|
5282
|
+
for (const auto& u : distinct) listed += (listed.empty() ? "" : ", ") + ("\"" + u + "\"");
|
|
5283
|
+
m.units_mixed[ch] = distinct;
|
|
5284
|
+
m.notes.push_back("channel " + ch + " has different units in the data the result keeps (" +
|
|
5285
|
+
listed + "); every segment keeps its own units string, samples are not "
|
|
5286
|
+
"rescaled, and the channel's label shows the first kept segment's (\"" +
|
|
5287
|
+
distinct.front() + "\") (validator units.mixed)");
|
|
5288
|
+
}
|
|
5289
|
+
}
|
|
5131
5290
|
m.pieces[ch] = std::move(pieces);
|
|
5132
5291
|
}
|
|
5133
5292
|
|
|
@@ -5587,7 +5746,8 @@ VerifyReport verify_merge_impl(MergePlan& m, const std::string& dst_path, const
|
|
|
5587
5746
|
const Report dr = handles ? detail::validate_session_through(handles->open(dst_path), dst_path, vo, inner)
|
|
5588
5747
|
: validate_session(dst_path, vo, inner);
|
|
5589
5748
|
for (const auto& f : dr.findings)
|
|
5590
|
-
if (f.severity != Severity::Info
|
|
5749
|
+
if (f.severity != Severity::Info &&
|
|
5750
|
+
!(f.check_id == "units.mixed" && m.units_mixed.count(f.channel)))
|
|
5591
5751
|
problem(f.path, f.check_id, "the validator reports on the result: " + f.message);
|
|
5592
5752
|
for (const auto& sk : dr.skipped)
|
|
5593
5753
|
problem(sk.path, "validator", "could not check a segment of the result: " + sk.reason);
|
|
@@ -5614,6 +5774,7 @@ void run_merge(const std::vector<std::string>& sources, const std::string& dst_p
|
|
|
5614
5774
|
rep.channels_without_samples = m.without_samples;
|
|
5615
5775
|
rep.output_encrypted = out.encrypt;
|
|
5616
5776
|
rep.notes = m.notes;
|
|
5777
|
+
rep.units_mixed = m.units_mixed;
|
|
5617
5778
|
rep.segments_split = m.segments_split;
|
|
5618
5779
|
const SourcePlan* plan0 = m.first_plan();
|
|
5619
5780
|
std::optional<Redactor> red;
|