acprof 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acprof/__init__.py +3 -0
- acprof/__main__.py +5 -0
- acprof/_bundle/.dockerignore +20 -0
- acprof/_bundle/LICENSE +202 -0
- acprof/_bundle/NOTICE +43 -0
- acprof/_bundle/acprof/__init__.py +3 -0
- acprof/_bundle/acprof/__main__.py +5 -0
- acprof/_bundle/acprof/analysis/__init__.py +1 -0
- acprof/_bundle/acprof/analysis/audit.py +276 -0
- acprof/_bundle/acprof/analysis/comparison.py +364 -0
- acprof/_bundle/acprof/analysis/compatibility.py +223 -0
- acprof/_bundle/acprof/analysis/conditions.py +185 -0
- acprof/_bundle/acprof/analysis/independent_comparison.py +191 -0
- acprof/_bundle/acprof/analysis/latency_model.py +1106 -0
- acprof/_bundle/acprof/analysis/latency_report.py +577 -0
- acprof/_bundle/acprof/analysis/model.py +276 -0
- acprof/_bundle/acprof/analysis/precision.py +51 -0
- acprof/_bundle/acprof/analysis/uncertainty.py +144 -0
- acprof/_bundle/acprof/application/__init__.py +1 -0
- acprof/_bundle/acprof/application/download.py +14 -0
- acprof/_bundle/acprof/application/errors.py +5 -0
- acprof/_bundle/acprof/application/notifications.py +302 -0
- acprof/_bundle/acprof/application/preparation.py +333 -0
- acprof/_bundle/acprof/application/resources.py +66 -0
- acprof/_bundle/acprof/application/run.py +462 -0
- acprof/_bundle/acprof/application/terminal_log.py +136 -0
- acprof/_bundle/acprof/artifact_layout.py +261 -0
- acprof/_bundle/acprof/artifacts.py +161 -0
- acprof/_bundle/acprof/capabilities.py +404 -0
- acprof/_bundle/acprof/cli/__init__.py +1 -0
- acprof/_bundle/acprof/cli/audit.py +74 -0
- acprof/_bundle/acprof/cli/auto.py +41 -0
- acprof/_bundle/acprof/cli/backfill_compute.py +498 -0
- acprof/_bundle/acprof/cli/compare.py +34 -0
- acprof/_bundle/acprof/cli/coverage.py +70 -0
- acprof/_bundle/acprof/cli/doctor.py +47 -0
- acprof/_bundle/acprof/cli/inspect.py +101 -0
- acprof/_bundle/acprof/cli/load.py +190 -0
- acprof/_bundle/acprof/cli/main.py +64 -0
- acprof/_bundle/acprof/cli/model_store.py +22 -0
- acprof/_bundle/acprof/cli/plot.py +211 -0
- acprof/_bundle/acprof/cli/posthoc.py +113 -0
- acprof/_bundle/acprof/cli/probe.py +186 -0
- acprof/_bundle/acprof/cli/report.py +35 -0
- acprof/_bundle/acprof/cli/results.py +37 -0
- acprof/_bundle/acprof/cli/run.py +54 -0
- acprof/_bundle/acprof/cli/stats.py +94 -0
- acprof/_bundle/acprof/cli/tui.py +65 -0
- acprof/_bundle/acprof/config.py +177 -0
- acprof/_bundle/acprof/container/__init__.py +1 -0
- acprof/_bundle/acprof/container/compat/__init__.py +1 -0
- acprof/_bundle/acprof/container/compat/transformers_dynamic.py +54 -0
- acprof/_bundle/acprof/container/compat/transformers_model_types.json +1091 -0
- acprof/_bundle/acprof/container/compute_profile_runner.py +392 -0
- acprof/_bundle/acprof/container/download_model.py +165 -0
- acprof/_bundle/acprof/container/execution.py +73 -0
- acprof/_bundle/acprof/container/handlers/__init__.py +298 -0
- acprof/_bundle/acprof/container/handlers/audio.py +743 -0
- acprof/_bundle/acprof/container/handlers/custom_pipeline.py +74 -0
- acprof/_bundle/acprof/container/handlers/cv.py +321 -0
- acprof/_bundle/acprof/container/handlers/diffusion.py +728 -0
- acprof/_bundle/acprof/container/handlers/moss.py +64 -0
- acprof/_bundle/acprof/container/handlers/multimodal.py +609 -0
- acprof/_bundle/acprof/container/handlers/nlp.py +747 -0
- acprof/_bundle/acprof/container/handlers/onnx_tasks.py +231 -0
- acprof/_bundle/acprof/container/handlers/onnxruntime.py +90 -0
- acprof/_bundle/acprof/container/handlers/structured.py +185 -0
- acprof/_bundle/acprof/container/handlers/timeseries.py +178 -0
- acprof/_bundle/acprof/container/load_policy.py +81 -0
- acprof/_bundle/acprof/container/loading_quality.py +48 -0
- acprof/_bundle/acprof/container/local_pipeline.py +31 -0
- acprof/_bundle/acprof/container/model_files.py +280 -0
- acprof/_bundle/acprof/container/model_probe.py +80 -0
- acprof/_bundle/acprof/container/onnx_session.py +150 -0
- acprof/_bundle/acprof/container/runtime_manifest.py +150 -0
- acprof/_bundle/acprof/container/runtime_validate.py +156 -0
- acprof/_bundle/acprof/container/server.py +209 -0
- acprof/_bundle/acprof/container/torch_execution.py +55 -0
- acprof/_bundle/acprof/container/validation.py +172 -0
- acprof/_bundle/acprof/cpu_affinity.py +35 -0
- acprof/_bundle/acprof/dependency_locks.py +129 -0
- acprof/_bundle/acprof/experiment.py +356 -0
- acprof/_bundle/acprof/extensions/__init__.py +294 -0
- acprof/_bundle/acprof/extensions/builtin/manifest.json +3591 -0
- acprof/_bundle/acprof/extensions/onnxruntime/manifest.json +191 -0
- acprof/_bundle/acprof/extensions/schema.py +238 -0
- acprof/_bundle/acprof/extensions/transformers/4.57.6.json +1639 -0
- acprof/_bundle/acprof/extensions/transformers/5.10.4.json +1812 -0
- acprof/_bundle/acprof/failures.py +115 -0
- acprof/_bundle/acprof/hardware_conditions.py +18 -0
- acprof/_bundle/acprof/hf_download.py +143 -0
- acprof/_bundle/acprof/hf_endpoints.py +62 -0
- acprof/_bundle/acprof/hf_transport.py +238 -0
- acprof/_bundle/acprof/host/__init__.py +1 -0
- acprof/_bundle/acprof/host/automation.py +161 -0
- acprof/_bundle/acprof/host/client.py +520 -0
- acprof/_bundle/acprof/host/collection_history.py +77 -0
- acprof/_bundle/acprof/host/collection_workflow.py +285 -0
- acprof/_bundle/acprof/host/command.py +119 -0
- acprof/_bundle/acprof/host/container_lifecycle.py +225 -0
- acprof/_bundle/acprof/host/container_state.py +128 -0
- acprof/_bundle/acprof/host/coverage_state.py +259 -0
- acprof/_bundle/acprof/host/detect.py +669 -0
- acprof/_bundle/acprof/host/docker_runtime.py +359 -0
- acprof/_bundle/acprof/host/doctor.py +158 -0
- acprof/_bundle/acprof/host/env_utils.py +233 -0
- acprof/_bundle/acprof/host/execution_conditions.py +87 -0
- acprof/_bundle/acprof/host/gpu_device.py +66 -0
- acprof/_bundle/acprof/host/hardware_conditions.py +130 -0
- acprof/_bundle/acprof/host/images/__init__.py +1 -0
- acprof/_bundle/acprof/host/images/dependencies.py +169 -0
- acprof/_bundle/acprof/host/images/environment.py +313 -0
- acprof/_bundle/acprof/host/images/graph.py +143 -0
- acprof/_bundle/acprof/host/images/identity.py +6 -0
- acprof/_bundle/acprof/host/images/inventory.py +445 -0
- acprof/_bundle/acprof/host/images/runtime.py +457 -0
- acprof/_bundle/acprof/host/images/scan.py +154 -0
- acprof/_bundle/acprof/host/input_plan.py +1056 -0
- acprof/_bundle/acprof/host/interface_probe.py +141 -0
- acprof/_bundle/acprof/host/largest_scale_probe.py +566 -0
- acprof/_bundle/acprof/host/matrix_plan.py +98 -0
- acprof/_bundle/acprof/host/measurement/__init__.py +1 -0
- acprof/_bundle/acprof/host/measurement/boundary.py +162 -0
- acprof/_bundle/acprof/host/measurement/capture.py +139 -0
- acprof/_bundle/acprof/host/measurement/checkpoint.py +275 -0
- acprof/_bundle/acprof/host/measurement/config.py +212 -0
- acprof/_bundle/acprof/host/measurement/diagnostics.py +247 -0
- acprof/_bundle/acprof/host/measurement/energy_validation.py +172 -0
- acprof/_bundle/acprof/host/measurement/errors.py +25 -0
- acprof/_bundle/acprof/host/measurement/execution.py +396 -0
- acprof/_bundle/acprof/host/measurement/load.py +237 -0
- acprof/_bundle/acprof/host/measurement/metrics.py +975 -0
- acprof/_bundle/acprof/host/measurement/monitors.py +143 -0
- acprof/_bundle/acprof/host/measurement/publication.py +171 -0
- acprof/_bundle/acprof/host/measurement/requests.py +97 -0
- acprof/_bundle/acprof/host/measurement/types.py +82 -0
- acprof/_bundle/acprof/host/model_coverage.py +337 -0
- acprof/_bundle/acprof/host/model_errors.py +116 -0
- acprof/_bundle/acprof/host/model_inspection.py +107 -0
- acprof/_bundle/acprof/host/model_schema.py +35 -0
- acprof/_bundle/acprof/host/model_store.py +733 -0
- acprof/_bundle/acprof/host/network_preflight.py +229 -0
- acprof/_bundle/acprof/host/orchestrator.py +1133 -0
- acprof/_bundle/acprof/host/permissions.py +117 -0
- acprof/_bundle/acprof/host/platform_metadata.py +63 -0
- acprof/_bundle/acprof/host/posthoc/__init__.py +12 -0
- acprof/_bundle/acprof/host/posthoc/backfill.py +415 -0
- acprof/_bundle/acprof/host/posthoc/context.py +400 -0
- acprof/_bundle/acprof/host/posthoc/plans.py +597 -0
- acprof/_bundle/acprof/host/posthoc/service.py +303 -0
- acprof/_bundle/acprof/host/posthoc/storage.py +491 -0
- acprof/_bundle/acprof/host/preflight.py +370 -0
- acprof/_bundle/acprof/host/profiling/__init__.py +1 -0
- acprof/_bundle/acprof/host/profiling/compute.py +436 -0
- acprof/_bundle/acprof/host/profiling/compute_plan.py +261 -0
- acprof/_bundle/acprof/host/profiling/execution.py +547 -0
- acprof/_bundle/acprof/host/profiling/execution_plan.py +576 -0
- acprof/_bundle/acprof/host/profiling/progress.py +83 -0
- acprof/_bundle/acprof/host/profiling/support.py +252 -0
- acprof/_bundle/acprof/host/profiling/tools/__init__.py +1 -0
- acprof/_bundle/acprof/host/profiling/tools/advisor.py +164 -0
- acprof/_bundle/acprof/host/profiling/tools/compute_parsers.py +409 -0
- acprof/_bundle/acprof/host/profiling/tools/execution_environment.py +135 -0
- acprof/_bundle/acprof/host/profiling/tools/execution_parsers.py +364 -0
- acprof/_bundle/acprof/host/profiling/tools/massif.py +418 -0
- acprof/_bundle/acprof/host/profiling/tools/ncu.py +696 -0
- acprof/_bundle/acprof/host/profiling/tools/nsys.py +326 -0
- acprof/_bundle/acprof/host/profiling/tools/tool_discovery.py +229 -0
- acprof/_bundle/acprof/host/profiling/tools/torch.py +158 -0
- acprof/_bundle/acprof/host/result_cleanup.py +65 -0
- acprof/_bundle/acprof/host/run_state.py +639 -0
- acprof/_bundle/acprof/host/runtime_settings.py +10 -0
- acprof/_bundle/acprof/host/runtime_validation.py +302 -0
- acprof/_bundle/acprof/host/source_bundle.py +96 -0
- acprof/_bundle/acprof/host/startup_probe.py +95 -0
- acprof/_bundle/acprof/host/static_metadata.py +936 -0
- acprof/_bundle/acprof/host/task_support.py +121 -0
- acprof/_bundle/acprof/installation.py +26 -0
- acprof/_bundle/acprof/latency_slo.py +62 -0
- acprof/_bundle/acprof/messages.py +26 -0
- acprof/_bundle/acprof/metric_registry.py +315 -0
- acprof/_bundle/acprof/model_candidates.py +297 -0
- acprof/_bundle/acprof/model_contract.py +234 -0
- acprof/_bundle/acprof/model_dependencies.py +93 -0
- acprof/_bundle/acprof/model_dependency_domain.py +45 -0
- acprof/_bundle/acprof/model_dependency_eval.py +438 -0
- acprof/_bundle/acprof/model_dependency_flow.py +200 -0
- acprof/_bundle/acprof/model_evidence.py +115 -0
- acprof/_bundle/acprof/model_metadata_analysis.py +156 -0
- acprof/_bundle/acprof/model_repository.py +149 -0
- acprof/_bundle/acprof/model_resolution.py +129 -0
- acprof/_bundle/acprof/model_review.py +155 -0
- acprof/_bundle/acprof/model_source_analysis.py +299 -0
- acprof/_bundle/acprof/model_spec.py +204 -0
- acprof/_bundle/acprof/model_support.py +105 -0
- acprof/_bundle/acprof/model_transforms.py +81 -0
- acprof/_bundle/acprof/monitors/__init__.py +1 -0
- acprof/_bundle/acprof/monitors/common.py +63 -0
- acprof/_bundle/acprof/monitors/energy_cpu.py +651 -0
- acprof/_bundle/acprof/monitors/energy_nvml.py +381 -0
- acprof/_bundle/acprof/monitors/gpu_metrics.py +123 -0
- acprof/_bundle/acprof/monitors/interfaces.py +74 -0
- acprof/_bundle/acprof/monitors/perf_mips.py +617 -0
- acprof/_bundle/acprof/monitors/rapl_topology.py +100 -0
- acprof/_bundle/acprof/monitors/resource_metrics.py +472 -0
- acprof/_bundle/acprof/monitors/resource_readers.py +435 -0
- acprof/_bundle/acprof/monitors/resource_usage.py +450 -0
- acprof/_bundle/acprof/monitors/sampling.py +79 -0
- acprof/_bundle/acprof/network_policy.py +182 -0
- acprof/_bundle/acprof/notifications.py +279 -0
- acprof/_bundle/acprof/packet/__init__.py +1 -0
- acprof/_bundle/acprof/packet/merge_packet_latency.py +369 -0
- acprof/_bundle/acprof/packet/sniff_parse_pcap.py +240 -0
- acprof/_bundle/acprof/pixel_metrics.py +103 -0
- acprof/_bundle/acprof/platform.py +180 -0
- acprof/_bundle/acprof/plotting/__init__.py +1 -0
- acprof/_bundle/acprof/plotting/config.py +487 -0
- acprof/_bundle/acprof/plotting/data.py +236 -0
- acprof/_bundle/acprof/plotting/diagnostics.py +747 -0
- acprof/_bundle/acprof/plotting/latency.py +638 -0
- acprof/_bundle/acprof/plotting/metrics.py +648 -0
- acprof/_bundle/acprof/plotting/plotly-LICENSE.txt +21 -0
- acprof/_bundle/acprof/plotting/report.py +66 -0
- acprof/_bundle/acprof/plotting/styles.py +111 -0
- acprof/_bundle/acprof/precision.py +42 -0
- acprof/_bundle/acprof/preparation_events.py +78 -0
- acprof/_bundle/acprof/progress_events.py +105 -0
- acprof/_bundle/acprof/quality.py +153 -0
- acprof/_bundle/acprof/resource_budget.py +29 -0
- acprof/_bundle/acprof/result_csv.py +263 -0
- acprof/_bundle/acprof/result_layers.py +365 -0
- acprof/_bundle/acprof/run_args.py +223 -0
- acprof/_bundle/acprof/run_options.py +225 -0
- acprof/_bundle/acprof/runtime_dependencies.py +195 -0
- acprof/_bundle/acprof/runtime_profiles.py +258 -0
- acprof/_bundle/acprof/runtime_settings.py +72 -0
- acprof/_bundle/acprof/source_identity.py +68 -0
- acprof/_bundle/acprof/telemetry.py +225 -0
- acprof/_bundle/acprof/testing/__init__.py +1 -0
- acprof/_bundle/acprof/testing/evidence.py +117 -0
- acprof/_bundle/acprof/testing/plugin.py +126 -0
- acprof/_bundle/acprof/testing/provenance.py +53 -0
- acprof/_bundle/acprof/testing/sharding.py +14 -0
- acprof/_bundle/acprof/tui/__init__.py +1 -0
- acprof/_bundle/acprof/tui/app.py +934 -0
- acprof/_bundle/acprof/tui/app_contracts.py +43 -0
- acprof/_bundle/acprof/tui/catalog_controller.py +120 -0
- acprof/_bundle/acprof/tui/commands.py +209 -0
- acprof/_bundle/acprof/tui/configuration_controller.py +436 -0
- acprof/_bundle/acprof/tui/diagnostics.py +340 -0
- acprof/_bundle/acprof/tui/downloads.py +56 -0
- acprof/_bundle/acprof/tui/environment.py +248 -0
- acprof/_bundle/acprof/tui/experiment_catalog.py +279 -0
- acprof/_bundle/acprof/tui/experiment_controller.py +132 -0
- acprof/_bundle/acprof/tui/experiment_picker.py +273 -0
- acprof/_bundle/acprof/tui/field_validation.py +62 -0
- acprof/_bundle/acprof/tui/i18n.py +57 -0
- acprof/_bundle/acprof/tui/image_controller.py +525 -0
- acprof/_bundle/acprof/tui/images.py +737 -0
- acprof/_bundle/acprof/tui/input.py +191 -0
- acprof/_bundle/acprof/tui/localization_controller.py +110 -0
- acprof/_bundle/acprof/tui/log.py +310 -0
- acprof/_bundle/acprof/tui/model_candidates.py +278 -0
- acprof/_bundle/acprof/tui/model_controller.py +111 -0
- acprof/_bundle/acprof/tui/model_input.py +12 -0
- acprof/_bundle/acprof/tui/model_store.py +166 -0
- acprof/_bundle/acprof/tui/preflight_controller.py +271 -0
- acprof/_bundle/acprof/tui/preparation.py +269 -0
- acprof/_bundle/acprof/tui/presentation.py +46 -0
- acprof/_bundle/acprof/tui/process.py +111 -0
- acprof/_bundle/acprof/tui/process_controller.py +218 -0
- acprof/_bundle/acprof/tui/profile_controller.py +109 -0
- acprof/_bundle/acprof/tui/progress.py +195 -0
- acprof/_bundle/acprof/tui/recovery.py +114 -0
- acprof/_bundle/acprof/tui/recovery_controller.py +169 -0
- acprof/_bundle/acprof/tui/rendering.py +75 -0
- acprof/_bundle/acprof/tui/reports.py +303 -0
- acprof/_bundle/acprof/tui/result_controller.py +279 -0
- acprof/_bundle/acprof/tui/review_inputs.py +54 -0
- acprof/_bundle/acprof/tui/run_form.py +125 -0
- acprof/_bundle/acprof/tui/run_planning.py +97 -0
- acprof/_bundle/acprof/tui/run_results.py +175 -0
- acprof/_bundle/acprof/tui/scrollbar.py +65 -0
- acprof/_bundle/acprof/tui/settings.py +203 -0
- acprof/_bundle/acprof/tui/slash_controller.py +95 -0
- acprof/_bundle/acprof/tui/stages.py +97 -0
- acprof/_bundle/acprof/tui/storage.py +150 -0
- acprof/_bundle/acprof/tui/table.py +213 -0
- acprof/_bundle/acprof/tui/themes.py +112 -0
- acprof/_bundle/acprof/tui/translation_catalog.py +1861 -0
- acprof/_bundle/acprof/tui/tui.tcss +373 -0
- acprof/_bundle/acprof/tui/views.py +706 -0
- acprof/_bundle/acprof/window_records.py +165 -0
- acprof/_bundle/acprof/workloads/__init__.py +222 -0
- acprof/_bundle/acprof/workloads/audio.py +621 -0
- acprof/_bundle/acprof/workloads/contract.py +116 -0
- acprof/_bundle/acprof/workloads/cv.py +240 -0
- acprof/_bundle/acprof/workloads/diffusion.py +410 -0
- acprof/_bundle/acprof/workloads/multimodal.py +329 -0
- acprof/_bundle/acprof/workloads/nlp.py +151 -0
- acprof/_bundle/acprof/workloads/structured.py +151 -0
- acprof/_bundle/acprof/workloads/timeseries.py +69 -0
- acprof/_bundle/assets/audio/librispeech-clean-test-en-30s/audio.wav +0 -0
- acprof/_bundle/assets/audio/librispeech-clean-test-en-30s/source.json +81 -0
- acprof/_bundle/dockerfiles/environment_tools.py +314 -0
- acprof/_bundle/dockerfiles/locks/audio-cpu.artifacts.json +503 -0
- acprof/_bundle/dockerfiles/locks/audio-cpu.txt +143 -0
- acprof/_bundle/dockerfiles/locks/audio-cu124.artifacts.json +600 -0
- acprof/_bundle/dockerfiles/locks/audio-cu124.txt +171 -0
- acprof/_bundle/dockerfiles/locks/audio-cu128.artifacts.json +544 -0
- acprof/_bundle/dockerfiles/locks/audio-cu128.txt +155 -0
- acprof/_bundle/dockerfiles/locks/custom-multimodal-cpu.artifacts.json +510 -0
- acprof/_bundle/dockerfiles/locks/custom-multimodal-cpu.txt +145 -0
- acprof/_bundle/dockerfiles/locks/custom-multimodal-cu124.artifacts.json +607 -0
- acprof/_bundle/dockerfiles/locks/custom-multimodal-cu124.txt +173 -0
- acprof/_bundle/dockerfiles/locks/custom-multimodal-cu128.artifacts.json +642 -0
- acprof/_bundle/dockerfiles/locks/custom-multimodal-cu128.txt +183 -0
- acprof/_bundle/dockerfiles/locks/cv-cpu.artifacts.json +279 -0
- acprof/_bundle/dockerfiles/locks/cv-cpu.txt +79 -0
- acprof/_bundle/dockerfiles/locks/cv-cu124.artifacts.json +376 -0
- acprof/_bundle/dockerfiles/locks/cv-cu124.txt +107 -0
- acprof/_bundle/dockerfiles/locks/cv-cu128.artifacts.json +411 -0
- acprof/_bundle/dockerfiles/locks/cv-cu128.txt +117 -0
- acprof/_bundle/dockerfiles/locks/diffusion-cpu.artifacts.json +335 -0
- acprof/_bundle/dockerfiles/locks/diffusion-cpu.txt +95 -0
- acprof/_bundle/dockerfiles/locks/diffusion-cu124.artifacts.json +432 -0
- acprof/_bundle/dockerfiles/locks/diffusion-cu124.txt +123 -0
- acprof/_bundle/dockerfiles/locks/diffusion-cu128.artifacts.json +467 -0
- acprof/_bundle/dockerfiles/locks/diffusion-cu128.txt +133 -0
- acprof/_bundle/dockerfiles/locks/moss-transformers5104.artifacts.json +621 -0
- acprof/_bundle/dockerfiles/locks/moss-transformers5104.txt +177 -0
- acprof/_bundle/dockerfiles/locks/multimodal-transformers4576.artifacts.json +635 -0
- acprof/_bundle/dockerfiles/locks/multimodal-transformers4576.txt +181 -0
- acprof/_bundle/dockerfiles/locks/nlp-cpu.artifacts.json +363 -0
- acprof/_bundle/dockerfiles/locks/nlp-cpu.txt +103 -0
- acprof/_bundle/dockerfiles/locks/nlp-cu124.artifacts.json +460 -0
- acprof/_bundle/dockerfiles/locks/nlp-cu124.txt +131 -0
- acprof/_bundle/dockerfiles/locks/nlp-cu128.artifacts.json +495 -0
- acprof/_bundle/dockerfiles/locks/nlp-cu128.txt +141 -0
- acprof/_bundle/dockerfiles/locks/onnxruntime-cpu.artifacts.json +261 -0
- acprof/_bundle/dockerfiles/locks/onnxruntime-cpu.txt +73 -0
- acprof/_bundle/dockerfiles/locks/platform-cpu.artifacts.json +99 -0
- acprof/_bundle/dockerfiles/locks/platform-cpu.txt +27 -0
- acprof/_bundle/dockerfiles/locks/platform-cu124.artifacts.json +196 -0
- acprof/_bundle/dockerfiles/locks/platform-cu124.txt +55 -0
- acprof/_bundle/dockerfiles/locks/platform-cu128.artifacts.json +231 -0
- acprof/_bundle/dockerfiles/locks/platform-cu128.txt +65 -0
- acprof/_bundle/dockerfiles/locks/platform-python-cpu.artifacts.json +37 -0
- acprof/_bundle/dockerfiles/locks/platform-python-cpu.txt +9 -0
- acprof/_bundle/dockerfiles/locks/structured-cpu.artifacts.json +279 -0
- acprof/_bundle/dockerfiles/locks/structured-cpu.txt +79 -0
- acprof/_bundle/dockerfiles/locks/structured-cu124.artifacts.json +376 -0
- acprof/_bundle/dockerfiles/locks/structured-cu124.txt +107 -0
- acprof/_bundle/dockerfiles/locks/structured-cu128.artifacts.json +411 -0
- acprof/_bundle/dockerfiles/locks/structured-cu128.txt +117 -0
- acprof/_bundle/dockerfiles/locks/system-trixie-amd64.json +620 -0
- acprof/_bundle/dockerfiles/locks/timeseries-cpu.artifacts.json +314 -0
- acprof/_bundle/dockerfiles/locks/timeseries-cpu.txt +89 -0
- acprof/_bundle/dockerfiles/locks/timeseries-cu124.artifacts.json +411 -0
- acprof/_bundle/dockerfiles/locks/timeseries-cu124.txt +117 -0
- acprof/_bundle/dockerfiles/locks/timeseries-cu128.artifacts.json +446 -0
- acprof/_bundle/dockerfiles/locks/timeseries-cu128.txt +127 -0
- acprof/_bundle/dockerfiles/locks/transformers5104-cpu.artifacts.json +594 -0
- acprof/_bundle/dockerfiles/locks/transformers5104-cpu.txt +169 -0
- acprof/_bundle/dockerfiles/locks/transformers5104-cu124.artifacts.json +691 -0
- acprof/_bundle/dockerfiles/locks/transformers5104-cu124.txt +197 -0
- acprof/_bundle/dockerfiles/locks/transformers5104-cu128.artifacts.json +726 -0
- acprof/_bundle/dockerfiles/locks/transformers5104-cu128.txt +207 -0
- acprof/_bundle/dockerfiles/platform.Dockerfile +22 -0
- acprof/_bundle/dockerfiles/requirements/audio.in +5 -0
- acprof/_bundle/dockerfiles/requirements/custom-multimodal.in +2 -0
- acprof/_bundle/dockerfiles/requirements/cv.in +4 -0
- acprof/_bundle/dockerfiles/requirements/diffusion.in +6 -0
- acprof/_bundle/dockerfiles/requirements/moss-transformers5104.in +9 -0
- acprof/_bundle/dockerfiles/requirements/multimodal-transformers4576.in +3 -0
- acprof/_bundle/dockerfiles/requirements/nlp.in +7 -0
- acprof/_bundle/dockerfiles/requirements/onnxruntime.in +9 -0
- acprof/_bundle/dockerfiles/requirements/service.in +8 -0
- acprof/_bundle/dockerfiles/requirements/structured.in +4 -0
- acprof/_bundle/dockerfiles/requirements/timeseries.in +2 -0
- acprof/_bundle/dockerfiles/requirements/timm.in +1 -0
- acprof/_bundle/dockerfiles/requirements/transformers5104.in +4 -0
- acprof/_bundle/dockerfiles/runtime-final.Dockerfile +34 -0
- acprof/_bundle/dockerfiles/runtime-model.Dockerfile +19 -0
- acprof/_bundle/dockerfiles/runtime.Dockerfile +23 -0
- acprof/_bundle/examples/coverage/regression.json +17 -0
- acprof/_bundle/examples/hardware-validation.json +19 -0
- acprof/_bundle/examples/multimodal/ultravox.model.json +29 -0
- acprof/_bundle/examples/nlp/smoke.py +111 -0
- acprof/_bundle/examples/onnxruntime/fixtures.py +164 -0
- acprof/_bundle/examples/onnxruntime/iris.json +16 -0
- acprof/_bundle/examples/onnxruntime/iris.model.json +7 -0
- acprof/_bundle/examples/onnxruntime/real_models.py +155 -0
- acprof/_bundle/examples/onnxruntime/smoke.py +68 -0
- acprof/_bundle/examples/structured/chronos_smoke.py +56 -0
- acprof/_bundle/examples/structured/export_models.py +71 -0
- acprof/_bundle/examples/structured/smoke.py +71 -0
- acprof/_bundle/licenses/CC-BY-4.0.txt +156 -0
- acprof/analysis/__init__.py +1 -0
- acprof/analysis/audit.py +276 -0
- acprof/analysis/comparison.py +364 -0
- acprof/analysis/compatibility.py +223 -0
- acprof/analysis/conditions.py +185 -0
- acprof/analysis/independent_comparison.py +191 -0
- acprof/analysis/latency_model.py +1106 -0
- acprof/analysis/latency_report.py +577 -0
- acprof/analysis/model.py +276 -0
- acprof/analysis/precision.py +51 -0
- acprof/analysis/uncertainty.py +144 -0
- acprof/application/__init__.py +1 -0
- acprof/application/download.py +14 -0
- acprof/application/errors.py +5 -0
- acprof/application/notifications.py +302 -0
- acprof/application/preparation.py +333 -0
- acprof/application/resources.py +66 -0
- acprof/application/run.py +462 -0
- acprof/application/terminal_log.py +136 -0
- acprof/artifact_layout.py +261 -0
- acprof/artifacts.py +161 -0
- acprof/capabilities.py +404 -0
- acprof/cli/__init__.py +1 -0
- acprof/cli/audit.py +74 -0
- acprof/cli/auto.py +41 -0
- acprof/cli/backfill_compute.py +498 -0
- acprof/cli/compare.py +34 -0
- acprof/cli/coverage.py +70 -0
- acprof/cli/doctor.py +47 -0
- acprof/cli/inspect.py +101 -0
- acprof/cli/load.py +190 -0
- acprof/cli/main.py +64 -0
- acprof/cli/model_store.py +22 -0
- acprof/cli/plot.py +211 -0
- acprof/cli/posthoc.py +113 -0
- acprof/cli/probe.py +186 -0
- acprof/cli/report.py +35 -0
- acprof/cli/results.py +37 -0
- acprof/cli/run.py +54 -0
- acprof/cli/stats.py +94 -0
- acprof/cli/tui.py +65 -0
- acprof/config.py +177 -0
- acprof/container/__init__.py +1 -0
- acprof/container/compat/__init__.py +1 -0
- acprof/container/compat/transformers_dynamic.py +54 -0
- acprof/container/compat/transformers_model_types.json +1091 -0
- acprof/container/compute_profile_runner.py +392 -0
- acprof/container/download_model.py +165 -0
- acprof/container/execution.py +73 -0
- acprof/container/handlers/__init__.py +298 -0
- acprof/container/handlers/audio.py +743 -0
- acprof/container/handlers/custom_pipeline.py +74 -0
- acprof/container/handlers/cv.py +321 -0
- acprof/container/handlers/diffusion.py +728 -0
- acprof/container/handlers/moss.py +64 -0
- acprof/container/handlers/multimodal.py +609 -0
- acprof/container/handlers/nlp.py +747 -0
- acprof/container/handlers/onnx_tasks.py +231 -0
- acprof/container/handlers/onnxruntime.py +90 -0
- acprof/container/handlers/structured.py +185 -0
- acprof/container/handlers/timeseries.py +178 -0
- acprof/container/load_policy.py +81 -0
- acprof/container/loading_quality.py +48 -0
- acprof/container/local_pipeline.py +31 -0
- acprof/container/model_files.py +280 -0
- acprof/container/model_probe.py +80 -0
- acprof/container/onnx_session.py +150 -0
- acprof/container/runtime_manifest.py +150 -0
- acprof/container/runtime_validate.py +156 -0
- acprof/container/server.py +209 -0
- acprof/container/torch_execution.py +55 -0
- acprof/container/validation.py +172 -0
- acprof/cpu_affinity.py +35 -0
- acprof/dependency_locks.py +129 -0
- acprof/experiment.py +356 -0
- acprof/extensions/__init__.py +294 -0
- acprof/extensions/builtin/manifest.json +3591 -0
- acprof/extensions/onnxruntime/manifest.json +191 -0
- acprof/extensions/schema.py +238 -0
- acprof/extensions/transformers/4.57.6.json +1639 -0
- acprof/extensions/transformers/5.10.4.json +1812 -0
- acprof/failures.py +115 -0
- acprof/hardware_conditions.py +18 -0
- acprof/hf_download.py +143 -0
- acprof/hf_endpoints.py +62 -0
- acprof/hf_transport.py +238 -0
- acprof/host/__init__.py +1 -0
- acprof/host/automation.py +161 -0
- acprof/host/client.py +520 -0
- acprof/host/collection_history.py +77 -0
- acprof/host/collection_workflow.py +285 -0
- acprof/host/command.py +119 -0
- acprof/host/container_lifecycle.py +225 -0
- acprof/host/container_state.py +128 -0
- acprof/host/coverage_state.py +259 -0
- acprof/host/detect.py +669 -0
- acprof/host/docker_runtime.py +359 -0
- acprof/host/doctor.py +158 -0
- acprof/host/env_utils.py +233 -0
- acprof/host/execution_conditions.py +87 -0
- acprof/host/gpu_device.py +66 -0
- acprof/host/hardware_conditions.py +130 -0
- acprof/host/images/__init__.py +1 -0
- acprof/host/images/dependencies.py +169 -0
- acprof/host/images/environment.py +313 -0
- acprof/host/images/graph.py +143 -0
- acprof/host/images/identity.py +6 -0
- acprof/host/images/inventory.py +445 -0
- acprof/host/images/runtime.py +457 -0
- acprof/host/images/scan.py +154 -0
- acprof/host/input_plan.py +1056 -0
- acprof/host/interface_probe.py +141 -0
- acprof/host/largest_scale_probe.py +566 -0
- acprof/host/matrix_plan.py +98 -0
- acprof/host/measurement/__init__.py +1 -0
- acprof/host/measurement/boundary.py +162 -0
- acprof/host/measurement/capture.py +139 -0
- acprof/host/measurement/checkpoint.py +275 -0
- acprof/host/measurement/config.py +212 -0
- acprof/host/measurement/diagnostics.py +247 -0
- acprof/host/measurement/energy_validation.py +172 -0
- acprof/host/measurement/errors.py +25 -0
- acprof/host/measurement/execution.py +396 -0
- acprof/host/measurement/load.py +237 -0
- acprof/host/measurement/metrics.py +975 -0
- acprof/host/measurement/monitors.py +143 -0
- acprof/host/measurement/publication.py +171 -0
- acprof/host/measurement/requests.py +97 -0
- acprof/host/measurement/types.py +82 -0
- acprof/host/model_coverage.py +337 -0
- acprof/host/model_errors.py +116 -0
- acprof/host/model_inspection.py +107 -0
- acprof/host/model_schema.py +35 -0
- acprof/host/model_store.py +733 -0
- acprof/host/network_preflight.py +229 -0
- acprof/host/orchestrator.py +1133 -0
- acprof/host/permissions.py +117 -0
- acprof/host/platform_metadata.py +63 -0
- acprof/host/posthoc/__init__.py +12 -0
- acprof/host/posthoc/backfill.py +415 -0
- acprof/host/posthoc/context.py +400 -0
- acprof/host/posthoc/plans.py +597 -0
- acprof/host/posthoc/service.py +303 -0
- acprof/host/posthoc/storage.py +491 -0
- acprof/host/preflight.py +370 -0
- acprof/host/profiling/__init__.py +1 -0
- acprof/host/profiling/compute.py +436 -0
- acprof/host/profiling/compute_plan.py +261 -0
- acprof/host/profiling/execution.py +547 -0
- acprof/host/profiling/execution_plan.py +576 -0
- acprof/host/profiling/progress.py +83 -0
- acprof/host/profiling/support.py +252 -0
- acprof/host/profiling/tools/__init__.py +1 -0
- acprof/host/profiling/tools/advisor.py +164 -0
- acprof/host/profiling/tools/compute_parsers.py +409 -0
- acprof/host/profiling/tools/execution_environment.py +135 -0
- acprof/host/profiling/tools/execution_parsers.py +364 -0
- acprof/host/profiling/tools/massif.py +418 -0
- acprof/host/profiling/tools/ncu.py +696 -0
- acprof/host/profiling/tools/nsys.py +326 -0
- acprof/host/profiling/tools/tool_discovery.py +229 -0
- acprof/host/profiling/tools/torch.py +158 -0
- acprof/host/result_cleanup.py +65 -0
- acprof/host/run_state.py +639 -0
- acprof/host/runtime_settings.py +10 -0
- acprof/host/runtime_validation.py +302 -0
- acprof/host/source_bundle.py +96 -0
- acprof/host/startup_probe.py +95 -0
- acprof/host/static_metadata.py +936 -0
- acprof/host/task_support.py +121 -0
- acprof/installation.py +26 -0
- acprof/latency_slo.py +62 -0
- acprof/messages.py +26 -0
- acprof/metric_registry.py +315 -0
- acprof/model_candidates.py +297 -0
- acprof/model_contract.py +234 -0
- acprof/model_dependencies.py +93 -0
- acprof/model_dependency_domain.py +45 -0
- acprof/model_dependency_eval.py +438 -0
- acprof/model_dependency_flow.py +200 -0
- acprof/model_evidence.py +115 -0
- acprof/model_metadata_analysis.py +156 -0
- acprof/model_repository.py +149 -0
- acprof/model_resolution.py +129 -0
- acprof/model_review.py +155 -0
- acprof/model_source_analysis.py +299 -0
- acprof/model_spec.py +204 -0
- acprof/model_support.py +105 -0
- acprof/model_transforms.py +81 -0
- acprof/monitors/__init__.py +1 -0
- acprof/monitors/common.py +63 -0
- acprof/monitors/energy_cpu.py +651 -0
- acprof/monitors/energy_nvml.py +381 -0
- acprof/monitors/gpu_metrics.py +123 -0
- acprof/monitors/interfaces.py +74 -0
- acprof/monitors/perf_mips.py +617 -0
- acprof/monitors/rapl_topology.py +100 -0
- acprof/monitors/resource_metrics.py +472 -0
- acprof/monitors/resource_readers.py +435 -0
- acprof/monitors/resource_usage.py +450 -0
- acprof/monitors/sampling.py +79 -0
- acprof/network_policy.py +182 -0
- acprof/notifications.py +279 -0
- acprof/packet/__init__.py +1 -0
- acprof/packet/merge_packet_latency.py +369 -0
- acprof/packet/sniff_parse_pcap.py +240 -0
- acprof/pixel_metrics.py +103 -0
- acprof/platform.py +180 -0
- acprof/plotting/__init__.py +1 -0
- acprof/plotting/config.py +487 -0
- acprof/plotting/data.py +236 -0
- acprof/plotting/diagnostics.py +747 -0
- acprof/plotting/latency.py +638 -0
- acprof/plotting/metrics.py +648 -0
- acprof/plotting/plotly-LICENSE.txt +21 -0
- acprof/plotting/report.css +61 -0
- acprof/plotting/report.html +68 -0
- acprof/plotting/report.js +295 -0
- acprof/plotting/report.py +66 -0
- acprof/plotting/styles.py +111 -0
- acprof/plotting/view_model.js +145 -0
- acprof/precision.py +42 -0
- acprof/preparation_events.py +78 -0
- acprof/progress_events.py +105 -0
- acprof/quality.py +153 -0
- acprof/resource_budget.py +29 -0
- acprof/result_csv.py +263 -0
- acprof/result_layers.py +365 -0
- acprof/run_args.py +223 -0
- acprof/run_options.py +225 -0
- acprof/runtime_dependencies.py +195 -0
- acprof/runtime_profiles.py +258 -0
- acprof/runtime_settings.py +72 -0
- acprof/source_identity.py +68 -0
- acprof/telemetry.py +225 -0
- acprof/testing/__init__.py +1 -0
- acprof/testing/evidence.py +117 -0
- acprof/testing/plugin.py +126 -0
- acprof/testing/provenance.py +53 -0
- acprof/testing/sharding.py +14 -0
- acprof/tui/__init__.py +1 -0
- acprof/tui/app.py +934 -0
- acprof/tui/app_contracts.py +43 -0
- acprof/tui/catalog_controller.py +120 -0
- acprof/tui/commands.py +209 -0
- acprof/tui/configuration_controller.py +436 -0
- acprof/tui/diagnostics.py +340 -0
- acprof/tui/downloads.py +56 -0
- acprof/tui/environment.py +248 -0
- acprof/tui/experiment_catalog.py +279 -0
- acprof/tui/experiment_controller.py +132 -0
- acprof/tui/experiment_picker.py +273 -0
- acprof/tui/field_validation.py +62 -0
- acprof/tui/i18n.py +57 -0
- acprof/tui/image_controller.py +525 -0
- acprof/tui/images.py +737 -0
- acprof/tui/input.py +191 -0
- acprof/tui/localization_controller.py +110 -0
- acprof/tui/log.py +310 -0
- acprof/tui/model_candidates.py +278 -0
- acprof/tui/model_controller.py +111 -0
- acprof/tui/model_input.py +12 -0
- acprof/tui/model_store.py +166 -0
- acprof/tui/preflight_controller.py +271 -0
- acprof/tui/preparation.py +269 -0
- acprof/tui/presentation.py +46 -0
- acprof/tui/process.py +111 -0
- acprof/tui/process_controller.py +218 -0
- acprof/tui/profile_controller.py +109 -0
- acprof/tui/progress.py +195 -0
- acprof/tui/recovery.py +114 -0
- acprof/tui/recovery_controller.py +169 -0
- acprof/tui/rendering.py +75 -0
- acprof/tui/reports.py +303 -0
- acprof/tui/result_controller.py +279 -0
- acprof/tui/review_inputs.py +54 -0
- acprof/tui/run_form.py +125 -0
- acprof/tui/run_planning.py +97 -0
- acprof/tui/run_results.py +175 -0
- acprof/tui/scrollbar.py +65 -0
- acprof/tui/settings.py +203 -0
- acprof/tui/slash_controller.py +95 -0
- acprof/tui/stages.py +97 -0
- acprof/tui/storage.py +150 -0
- acprof/tui/table.py +213 -0
- acprof/tui/themes.py +112 -0
- acprof/tui/translation_catalog.py +1861 -0
- acprof/tui/tui.tcss +373 -0
- acprof/tui/views.py +706 -0
- acprof/window_records.py +165 -0
- acprof/workloads/__init__.py +222 -0
- acprof/workloads/audio.py +621 -0
- acprof/workloads/contract.py +116 -0
- acprof/workloads/cv.py +240 -0
- acprof/workloads/diffusion.py +410 -0
- acprof/workloads/multimodal.py +329 -0
- acprof/workloads/nlp.py +151 -0
- acprof/workloads/structured.py +151 -0
- acprof/workloads/timeseries.py +69 -0
- acprof-0.3.0.dist-info/METADATA +117 -0
- acprof-0.3.0.dist-info/RECORD +704 -0
- acprof-0.3.0.dist-info/WHEEL +4 -0
- acprof-0.3.0.dist-info/entry_points.txt +2 -0
- acprof-0.3.0.dist-info/licenses/LICENSE +202 -0
- acprof-0.3.0.dist-info/licenses/NOTICE +43 -0
- acprof-0.3.0.dist-info/licenses/licenses/CC-BY-4.0.txt +156 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Portable, purpose-specific condition evidence for Python and offline reports.
|
|
2
|
+
|
|
3
|
+
Task and resource policy lives here. Consumers only compare recorded tokens and
|
|
4
|
+
exact rational distributions; missing evidence never becomes an equality proof.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
from collections import defaultdict
|
|
11
|
+
from fractions import Fraction
|
|
12
|
+
|
|
13
|
+
PURPOSES = ("same-hardware", "cross-hardware", "resource-scaling")
|
|
14
|
+
ALLOWED_RESOURCE_DIMENSIONS = ["cpu", "memory"]
|
|
15
|
+
_FIXED_OUTPUT_TASKS = {"tabular-regression", "tabular-classification", "time-series-forecasting"}
|
|
16
|
+
_DYNAMIC_GENERATION = {f"generation.{name}" for name in (
|
|
17
|
+
"actual_output_tokens", "actual_output_tokens_per_sequence", "actual_output_tokens_status",
|
|
18
|
+
"decoded_text_token_count", "stop_reason")}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def canonical(value):
|
|
22
|
+
return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def digest(value):
|
|
26
|
+
return hashlib.sha256(canonical(value).encode()).hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def unknown(value):
|
|
30
|
+
if value is None or value == "unknown":
|
|
31
|
+
return True
|
|
32
|
+
if isinstance(value, dict):
|
|
33
|
+
return not value or any(unknown(item) for item in value.values())
|
|
34
|
+
if isinstance(value, list):
|
|
35
|
+
return not value or any(unknown(item) for item in value)
|
|
36
|
+
return False
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def dimensions(contract, prefix=""):
|
|
40
|
+
result = {}
|
|
41
|
+
for key, value in contract.items():
|
|
42
|
+
name = f"{prefix}.{key}" if prefix else key
|
|
43
|
+
if isinstance(value, dict):
|
|
44
|
+
result.update(dimensions(value, name))
|
|
45
|
+
else:
|
|
46
|
+
result[name] = value
|
|
47
|
+
return result
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def workload_case_profile(case):
|
|
51
|
+
variants = []
|
|
52
|
+
distribution: defaultdict[str, Fraction] = defaultdict(Fraction)
|
|
53
|
+
complete = True
|
|
54
|
+
for variant in case["variants"]:
|
|
55
|
+
contract = variant["contract"]
|
|
56
|
+
weight = Fraction(variant["count"], case["request_count"])
|
|
57
|
+
distribution[canonical(contract)] += weight
|
|
58
|
+
fields = {name: canonical(value) for name, value in dimensions(contract).items() if not unknown(value)}
|
|
59
|
+
controlled = [name for name in fields if contract.get("task") in _FIXED_OUTPUT_TASKS or
|
|
60
|
+
((not name.startswith("output.") or name == "output.type") and name not in _DYNAMIC_GENERATION)]
|
|
61
|
+
variants.append({"proportion": str(weight), "fields": fields, "controlled": sorted(controlled)})
|
|
62
|
+
complete = complete and not unknown(contract)
|
|
63
|
+
return {"observed": digest({key: str(value) for key, value in sorted(distribution.items())}),
|
|
64
|
+
"complete": complete, "variants": variants}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def compare_workload_profiles(left, right):
|
|
68
|
+
if left is None or right is None:
|
|
69
|
+
return "unknown"
|
|
70
|
+
if left.keys() != right.keys():
|
|
71
|
+
return "incompatible"
|
|
72
|
+
status = "compatible"
|
|
73
|
+
for key in left:
|
|
74
|
+
a, b = left[key], right[key]
|
|
75
|
+
if a.get("ambiguous") or b.get("ambiguous"):
|
|
76
|
+
status = "unknown"
|
|
77
|
+
continue
|
|
78
|
+
variants = [*a["variants"], *b["variants"]]
|
|
79
|
+
known = set.intersection(*(set(item["fields"]) for item in variants))
|
|
80
|
+
|
|
81
|
+
def controlled_distribution(case):
|
|
82
|
+
values: defaultdict[str, Fraction] = defaultdict(Fraction)
|
|
83
|
+
for item in case["variants"]:
|
|
84
|
+
fields = {name: item["fields"][name] for name in item["controlled"] if name in known}
|
|
85
|
+
values[canonical(fields)] += Fraction(item["proportion"])
|
|
86
|
+
return values
|
|
87
|
+
|
|
88
|
+
if controlled_distribution(a) != controlled_distribution(b):
|
|
89
|
+
return "incompatible"
|
|
90
|
+
if not a["complete"] or not b["complete"] or a["observed"] != b["observed"]:
|
|
91
|
+
status = "unknown"
|
|
92
|
+
return status
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def comparison_profile(snapshot, *, purpose="same-hardware", case=None):
|
|
96
|
+
"""Project recorded evidence to a selected CSV case, without rereading files."""
|
|
97
|
+
if purpose not in PURPOSES:
|
|
98
|
+
raise ValueError(f"comparison purpose must be one of {PURPOSES}")
|
|
99
|
+
conditions = dict(snapshot["conditions"])
|
|
100
|
+
hardware = dict(snapshot["hardware"])
|
|
101
|
+
workload = conditions.pop("actual_workload")
|
|
102
|
+
if case is not None:
|
|
103
|
+
cpu, memory, gpu, scale = case
|
|
104
|
+
case_id = f"{int(float(cpu))}c_{int(float(memory))}g_{gpu}"
|
|
105
|
+
conditions["resources"] = {**conditions["resources"], "cpus": str(cpu), "mems": str(memory), "gpus": gpu}
|
|
106
|
+
entries = snapshot.get("planned_input_entries")
|
|
107
|
+
selected = [entry for entry in entries or [] if float(entry["input_scale"]) == float(scale)]
|
|
108
|
+
conditions["planned_inputs"] = digest(selected) if selected else None
|
|
109
|
+
selected_case = canonical(list(case))
|
|
110
|
+
workload = ({selected_case: workload[selected_case]} if workload and selected_case in workload else None)
|
|
111
|
+
hardware = {name: {case_id: values.get(case_id)} if isinstance(values, dict) else None
|
|
112
|
+
for name, values in hardware.items()}
|
|
113
|
+
if not unknown(hardware.get("runtime_threads")):
|
|
114
|
+
conditions["runtime_threads"] = hardware["runtime_threads"]
|
|
115
|
+
workload_profiles: dict[str, dict[str, object]] | None = None
|
|
116
|
+
if workload is not None:
|
|
117
|
+
workload_profiles = {}
|
|
118
|
+
for key, item in workload.items():
|
|
119
|
+
normalized_key = canonical(json.loads(key)[2:]) if purpose == "resource-scaling" else key
|
|
120
|
+
profile = workload_case_profile(item)
|
|
121
|
+
previous = workload_profiles.get(normalized_key)
|
|
122
|
+
if previous and previous != profile:
|
|
123
|
+
profile = {"ambiguous": True}
|
|
124
|
+
workload_profiles[normalized_key] = profile
|
|
125
|
+
if purpose == "resource-scaling":
|
|
126
|
+
# Resource labels in case maps are coordinates, not hardware facts.
|
|
127
|
+
# Different measured policies/threads across resource cases stay visible.
|
|
128
|
+
def remove_resource_keys(values):
|
|
129
|
+
if not isinstance(values, dict):
|
|
130
|
+
return values
|
|
131
|
+
by_device: dict[str, dict[str, object]] = {}
|
|
132
|
+
for key, value in values.items():
|
|
133
|
+
by_device.setdefault(key.rsplit("_", 1)[-1], {})[canonical(value)] = value
|
|
134
|
+
return {device: [items[key] for key in sorted(items)] for device, items in by_device.items()}
|
|
135
|
+
hardware = {name: remove_resource_keys(values) for name, values in hardware.items()}
|
|
136
|
+
if not unknown(snapshot["hardware"].get("runtime_threads")):
|
|
137
|
+
conditions["runtime_threads"] = hardware["runtime_threads"]
|
|
138
|
+
checks = {}
|
|
139
|
+
|
|
140
|
+
def add(name, value, difference="incompatible"):
|
|
141
|
+
if isinstance(value, dict) and value:
|
|
142
|
+
for key, item in value.items():
|
|
143
|
+
add(f"{name}.{key}", item, difference)
|
|
144
|
+
else:
|
|
145
|
+
checks[name] = {"value": None if unknown(value) else digest(value), "difference": difference}
|
|
146
|
+
|
|
147
|
+
for name, value in conditions.items():
|
|
148
|
+
if name == "resources" and purpose == "resource-scaling":
|
|
149
|
+
for key, item in value.items():
|
|
150
|
+
add(f"resources.{key}", item, "expected_difference" if key in {"cpus", "mems"} else "incompatible")
|
|
151
|
+
else:
|
|
152
|
+
add(name, value)
|
|
153
|
+
for name, value in hardware.items():
|
|
154
|
+
if purpose == "cross-hardware" and name != "runtime_threads":
|
|
155
|
+
# Whole maps are allowed to differ, but every recorded leaf must exist.
|
|
156
|
+
checks[f"hardware_{name}"] = {"value": None if unknown(value) else digest(value),
|
|
157
|
+
"difference": "expected_difference"}
|
|
158
|
+
else:
|
|
159
|
+
add(f"hardware_{name}", value)
|
|
160
|
+
profile = {"purpose": purpose, "valid": snapshot["valid"], "checks": checks, "workload": workload_profiles}
|
|
161
|
+
verdict = compare_profiles(profile, profile)
|
|
162
|
+
profile["status"] = verdict["status"]
|
|
163
|
+
profile["reasons"] = verdict["reasons"]
|
|
164
|
+
# Identical complete evidence forms an equivalence group. Expected differences
|
|
165
|
+
# do not split it; incomplete evidence cannot create an automatic candidate.
|
|
166
|
+
profile["cohort"] = digest({"checks": {key: item["value"] if item["difference"] != "expected_difference" else True
|
|
167
|
+
for key, item in checks.items()}, "workload": workload_profiles})
|
|
168
|
+
return profile
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def compare_profiles(left, right):
|
|
172
|
+
if left["purpose"] != right["purpose"]:
|
|
173
|
+
raise ValueError("comparison profile purposes differ")
|
|
174
|
+
statuses = {}
|
|
175
|
+
for key in left["checks"].keys() | right["checks"].keys():
|
|
176
|
+
a, b = left["checks"].get(key), right["checks"].get(key)
|
|
177
|
+
statuses[key] = ("unknown" if not a or not b or a["value"] is None or b["value"] is None else
|
|
178
|
+
"compatible" if a["value"] == b["value"] else a["difference"])
|
|
179
|
+
statuses["actual_workload"] = compare_workload_profiles(left["workload"], right["workload"])
|
|
180
|
+
if not left["valid"] or not right["valid"]:
|
|
181
|
+
statuses["invalid_artifacts"] = "incompatible"
|
|
182
|
+
status = ("incompatible" if "incompatible" in statuses.values() else
|
|
183
|
+
"unknown" if "unknown" in statuses.values() else "compatible")
|
|
184
|
+
return {"status": status, "reasons": sorted({name.split(".")[0] for name, value in statuses.items()
|
|
185
|
+
if value in {"incompatible", "unknown"}}), "checks": statuses}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Compare run means, resampling independent runs instead of individual requests."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
import random
|
|
6
|
+
import statistics
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from acprof.analysis.audit import number
|
|
12
|
+
from acprof.analysis.comparison import compare_results
|
|
13
|
+
from acprof.analysis.conditions import ALLOWED_RESOURCE_DIMENSIONS
|
|
14
|
+
from acprof.analysis.uncertainty import summarize_windows
|
|
15
|
+
from acprof.artifact_layout import ArtifactLayout
|
|
16
|
+
from acprof.artifacts import file_sha256, read_json_object
|
|
17
|
+
from acprof.hardware_conditions import conditions_path
|
|
18
|
+
from acprof.metric_registry import METRICS
|
|
19
|
+
from acprof.quality import QUALITY_FIELDS, combine_quality
|
|
20
|
+
from acprof.result_csv import measurement_key, read_result_csv
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _interval(values: list[float], confidence: float):
|
|
24
|
+
ordered = sorted(values)
|
|
25
|
+
|
|
26
|
+
def quantile(p):
|
|
27
|
+
position = (len(ordered) - 1) * p
|
|
28
|
+
lo = math.floor(position)
|
|
29
|
+
fraction = position - lo
|
|
30
|
+
return ordered[lo] * (1 - fraction) + ordered[min(lo + 1, len(ordered) - 1)] * fraction
|
|
31
|
+
|
|
32
|
+
tail = (1 - confidence) / 2
|
|
33
|
+
return [quantile(tail), quantile(1 - tail)]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _bootstrap(left, right, *, confidence, resamples, seed):
|
|
37
|
+
rng = random.Random(seed)
|
|
38
|
+
differences, ratios = [], []
|
|
39
|
+
for _ in range(resamples):
|
|
40
|
+
a = statistics.fmean(rng.choices(left, k=len(left)))
|
|
41
|
+
b = statistics.fmean(rng.choices(right, k=len(right)))
|
|
42
|
+
differences.append(b - a)
|
|
43
|
+
if a > 0:
|
|
44
|
+
ratios.append(b / a)
|
|
45
|
+
return (_interval(differences, confidence),
|
|
46
|
+
_interval(ratios, confidence) if len(ratios) == resamples else None)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _aggregate_evidence(records):
|
|
50
|
+
statuses = {record["run_status"] for record in records}
|
|
51
|
+
measurement = {record["measurement_status"] for record in records}
|
|
52
|
+
return {**combine_quality(records), "run_status": next(iter(statuses)) if len(statuses) == 1 else "mixed" if statuses else "unknown",
|
|
53
|
+
"measurement_status": ("incomplete" if "incomplete" in measurement else
|
|
54
|
+
"complete" if measurement == {"complete"} else "unknown")}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def compare_experiments(left, right, *, metrics, purpose="same-hardware",
|
|
58
|
+
confidence=0.95, resamples=5000, seed=0):
|
|
59
|
+
# Reuse the metric/window validation contract; no profiler or lifecycle reuse values.
|
|
60
|
+
summarize_windows([], metrics, confidence=confidence, resamples=resamples, seed=seed)
|
|
61
|
+
if not left or not right:
|
|
62
|
+
raise ValueError("both comparison groups require at least one experiment")
|
|
63
|
+
sources = {"left": list(left), "right": list(right)}
|
|
64
|
+
experiments: dict[str, list[dict[str, Any]]] = {}
|
|
65
|
+
run_ids: set[str] = set()
|
|
66
|
+
fingerprints: dict[Path, str | None] = {}
|
|
67
|
+
groups: defaultdict[tuple[str, ...], dict[str, list[dict[str, Any]]]] = defaultdict(
|
|
68
|
+
lambda: {"left": [], "right": []}
|
|
69
|
+
)
|
|
70
|
+
checks = []
|
|
71
|
+
for side, paths in sources.items():
|
|
72
|
+
experiments[side] = []
|
|
73
|
+
for path in paths:
|
|
74
|
+
source = Path(path)
|
|
75
|
+
layout = ArtifactLayout.discover(source) if source.is_dir() else ArtifactLayout.from_csv(source)
|
|
76
|
+
state = read_json_object(layout.path("run_state.json"), label="run state")
|
|
77
|
+
run_id = state.get("run_id")
|
|
78
|
+
if not isinstance(run_id, str) or not run_id:
|
|
79
|
+
raise ValueError("independent comparison requires recorded run IDs")
|
|
80
|
+
if run_id in run_ids:
|
|
81
|
+
raise ValueError(f"duplicate independent run: {run_id}")
|
|
82
|
+
run_ids.add(run_id)
|
|
83
|
+
# Detect concurrent changes to the rows and their condition evidence.
|
|
84
|
+
for name in ("result_layers.json", "run_state.json", "input_scale_plan.json",
|
|
85
|
+
"static_meta.json", "hardware_conditions.json", "quality_checks.json", "runtime_validation.json"):
|
|
86
|
+
artifact = (layout.result_csv if name == "result_layers.json" else
|
|
87
|
+
conditions_path(layout) if name == "hardware_conditions.json" else layout.path(name))
|
|
88
|
+
fingerprints[artifact] = file_sha256(artifact) if artifact.exists() else None
|
|
89
|
+
_, rows = read_result_csv(layout.result_csv)
|
|
90
|
+
summarize_windows(rows, metrics, confidence=confidence, resamples=1)
|
|
91
|
+
complete = state.get("status") == "complete"
|
|
92
|
+
cross = compare_results(left[0], path, purpose=purpose)
|
|
93
|
+
snapshot = cross["experiments"]["right"]
|
|
94
|
+
evidence = {key: snapshot[key] for key in (*QUALITY_FIELDS, "run_status", "measurement_status")}
|
|
95
|
+
experiments[side].append({"run_id": run_id, "source": str(source.resolve()),
|
|
96
|
+
"state": state.get("status"), "eligible": complete, **evidence})
|
|
97
|
+
checks.append({"left_run_id": next(iter(experiments["left"]))["run_id"],
|
|
98
|
+
"right_run_id": run_id, **cross})
|
|
99
|
+
within = compare_results(paths[0], path, purpose="same-hardware")
|
|
100
|
+
checks.append({**within, "group": side, "within_group": True, "run_id": run_id})
|
|
101
|
+
if within and within["expected_differences"]:
|
|
102
|
+
checks.append({"status": "incompatible", "run_id": run_id,
|
|
103
|
+
"reason": "replicates_change_experiment_identity",
|
|
104
|
+
"differences": within["expected_differences"]})
|
|
105
|
+
by_case = defaultdict(list)
|
|
106
|
+
for row in rows:
|
|
107
|
+
key = measurement_key(row)
|
|
108
|
+
if key[4] == "0":
|
|
109
|
+
by_case[key[:4]].append(row)
|
|
110
|
+
scaling_cases = set()
|
|
111
|
+
for case, windows in by_case.items():
|
|
112
|
+
comparison_case = case
|
|
113
|
+
if purpose == "resource-scaling":
|
|
114
|
+
if case[2:] in scaling_cases:
|
|
115
|
+
raise ValueError("resource-scaling requires one resource configuration per device/input in each run")
|
|
116
|
+
scaling_cases.add(case[2:])
|
|
117
|
+
comparison_case = ("0", "0", *case[2:])
|
|
118
|
+
for metric in metrics:
|
|
119
|
+
window_values: list[float] = []
|
|
120
|
+
for row in windows:
|
|
121
|
+
value = number(row.get(metric))
|
|
122
|
+
if row.get("status") == "ok" and value is not None:
|
|
123
|
+
window_values.append(value)
|
|
124
|
+
groups[(*comparison_case, metric)][side].append({
|
|
125
|
+
"run_id": run_id, "mean": statistics.fmean(window_values) if window_values and complete else None,
|
|
126
|
+
"resource_coordinates": {"cpu_cores": float(case[0]), "mem_cap_gb": float(case[1])},
|
|
127
|
+
"valid_windows": len(window_values), "failed_windows": sum(row.get("status") != "ok" for row in windows),
|
|
128
|
+
"missing_windows": sum(row.get("status") == "ok" and number(row.get(metric)) is None for row in windows),
|
|
129
|
+
"eligible": complete, **evidence,
|
|
130
|
+
})
|
|
131
|
+
statuses = {check["status"] for check in checks}
|
|
132
|
+
status = "incompatible" if "incompatible" in statuses else "unknown" if "unknown" in statuses else "compatible"
|
|
133
|
+
output = []
|
|
134
|
+
for (cpu, mem, gpu, scale, metric), sides in sorted(groups.items()):
|
|
135
|
+
coordinates = {side: records[0]["resource_coordinates"] if records else None for side, records in sides.items()}
|
|
136
|
+
baseline_coordinates = coordinates["left"] or coordinates["right"]
|
|
137
|
+
if purpose == "resource-scaling":
|
|
138
|
+
if baseline_coordinates is None:
|
|
139
|
+
raise ValueError("resource-scaling case has no recorded resource coordinates")
|
|
140
|
+
cpu, mem = baseline_coordinates["cpu_cores"], baseline_coordinates["mem_cap_gb"]
|
|
141
|
+
result: dict[str, Any] = {"cpu_cores": float(cpu), "mem_cap_gb": float(mem), "gpu_mode": gpu,
|
|
142
|
+
"resource_coordinates": coordinates,
|
|
143
|
+
"input_scale": float(scale), "metric": metric, "unit": METRICS[metric].unit,
|
|
144
|
+
"difference": None, "ratio": None, "difference_ci": None, "ratio_ci": None, "reason": ""}
|
|
145
|
+
blocked = any(check.get("metric_comparability", {}).get(metric, {}).get("status") == "not comparable"
|
|
146
|
+
for check in checks)
|
|
147
|
+
result["comparability"] = "not comparable" if blocked or status != "compatible" else "comparable"
|
|
148
|
+
values = {}
|
|
149
|
+
for side in ("left", "right"):
|
|
150
|
+
records = sides[side]
|
|
151
|
+
values[side] = [record["mean"] for record in records if record["mean"] is not None]
|
|
152
|
+
result[side] = {**_aggregate_evidence(records), "n_runs": len(values[side]), "requested_runs": len(sources[side]),
|
|
153
|
+
"missing_runs": len(sources[side]) - len(values[side]),
|
|
154
|
+
"mean": statistics.fmean(values[side]) if values[side] else None,
|
|
155
|
+
"std": statistics.stdev(values[side]) if len(values[side]) > 1 else None,
|
|
156
|
+
"failed_windows": sum(record["failed_windows"] for record in records),
|
|
157
|
+
"missing_windows": sum(record["missing_windows"] for record in records), "runs": records}
|
|
158
|
+
if blocked:
|
|
159
|
+
result["reason"] = "not comparable"
|
|
160
|
+
elif status != "compatible":
|
|
161
|
+
result["reason"] = "conditions_" + status
|
|
162
|
+
elif not values["left"] or not values["right"]:
|
|
163
|
+
result["reason"] = "missing_comparable_runs"
|
|
164
|
+
else:
|
|
165
|
+
a, b = result["left"]["mean"], result["right"]["mean"]
|
|
166
|
+
result.update(difference=b - a, ratio=b / a if a > 0 else None)
|
|
167
|
+
if min(len(values["left"]), len(values["right"])) < 3:
|
|
168
|
+
result["reason"] = "insufficient_independent_runs"
|
|
169
|
+
else:
|
|
170
|
+
result["difference_ci"], result["ratio_ci"] = _bootstrap(
|
|
171
|
+
values["left"], values["right"], confidence=confidence, resamples=resamples, seed=seed)
|
|
172
|
+
if result["ratio_ci"] is None:
|
|
173
|
+
result["reason"] = "nonpositive_baseline_for_ratio"
|
|
174
|
+
output.append(result)
|
|
175
|
+
for artifact, digest in fingerprints.items():
|
|
176
|
+
current = file_sha256(artifact) if artifact.exists() else None
|
|
177
|
+
if current != digest:
|
|
178
|
+
raise ValueError(f"experiment changed during comparison: {artifact}")
|
|
179
|
+
return {"schema_version": 1, "kind": "independent_experiment_comparison", "status": status,
|
|
180
|
+
"allowed_resource_dimensions": ALLOWED_RESOURCE_DIMENSIONS if purpose == "resource-scaling" else [],
|
|
181
|
+
"purpose": purpose, "confidence": confidence, "resamples": resamples, "seed": seed,
|
|
182
|
+
"resampling_unit": "independent_run_mean", "method": "unpaired_percentile_bootstrap",
|
|
183
|
+
"direction": "right_minus_left_and_right_divided_by_left", "groups": output,
|
|
184
|
+
"experiments": experiments, "condition_checks": checks,
|
|
185
|
+
"quality": {side: combine_quality(records) for side, records in experiments.items()},
|
|
186
|
+
"limitations": ["distinct run IDs and independent restarts are required; independence is an experimental assumption",
|
|
187
|
+
"resource-scaling allows CPU and memory quotas; simultaneous changes do not isolate either effect",
|
|
188
|
+
"successful windows are averaged equally within a run; run means are then weighted equally",
|
|
189
|
+
"intervals describe run variability, not continuous isolation, thermal equivalence or model quality",
|
|
190
|
+
"failed and missing windows are excluded and reported; success-conditioned estimates may be biased"],
|
|
191
|
+
"source_sha256": {str(path): digest for path, digest in fingerprints.items()}}
|