tracebloc 1.2.71__tar.gz → 1.2.72__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tracebloc-1.2.71 → tracebloc-1.2.72}/PKG-INFO +29 -6
- {tracebloc-1.2.71 → tracebloc-1.2.72}/README.md +4 -4
- {tracebloc-1.2.71 → tracebloc-1.2.72}/pyproject.toml +40 -11
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/_framework_utils.py +3 -2
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/_link_regeneration.py +71 -10
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/_stored_link.py +81 -5
- tracebloc-1.2.72/tracebloc/_upload_conversion.py +215 -0
- tracebloc-1.2.72/tracebloc/contracts/training_requirements.source.json +9 -0
- tracebloc-1.2.72/tracebloc/contracts/training_requirements.txt +56 -0
- tracebloc-1.2.72/tracebloc/convert/acceptance.py +375 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/families/hf/__init__.py +22 -1
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/families/hf/_build.py +71 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/gate.py +50 -16
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/interfaces.py +25 -0
- tracebloc-1.2.72/tracebloc/convert/licence/__init__.py +493 -0
- tracebloc-1.2.72/tracebloc/convert/licence/spdx.py +194 -0
- tracebloc-1.2.72/tracebloc/convert/outputs.py +433 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/checkpoint.py +16 -2
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/model_class.py +1 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/remote.py +11 -2
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/repo.py +29 -4
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/shake.py +86 -6
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/source_tree.py +23 -1
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/package.py +79 -3
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/report.py +7 -0
- tracebloc-1.2.72/tracebloc/convert/training_versions.py +162 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/fetch.py +42 -13
- tracebloc-1.2.72/tracebloc/library_fetch.py +110 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/raw_mode_torch.py +2 -7
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/seed_weights.py +1 -1
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/model_upload.py +9 -14
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/weight_loaders.py +2 -3
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/user.py +41 -18
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/safetensors_io.py +61 -3
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc.egg-info/PKG-INFO +29 -6
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc.egg-info/SOURCES.txt +9 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc.egg-info/requires.txt +29 -1
- {tracebloc-1.2.71 → tracebloc-1.2.72}/MANIFEST.in +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/license.txt +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/setup.cfg +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/_deprecation.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/_http_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/_link_answer.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/_linking.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/_raw_width_note.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/checks.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/client.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/config.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/contracts/stage_urls.v1.json +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/contracts/stage_urls.v1.source.json +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/contracts/survival_payload.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/families/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/families/tabular.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/families/vision/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/families/vision/_matching.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/arguments.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/assemble.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/gate_rules.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/generate.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/head.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/import_check.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/layer0.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/owncode/live.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/convert/tokenizers.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/env_login.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/exceptions.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/inference.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/lifelines_weights.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/link_model_dataset.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/notebook/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/notebook/templates/_partials.tpl +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/notebook/templates/embeddings.tpl +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/notebook/templates/nlp_finetune.tpl +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/notebook/templates/nlp_generative.tpl +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/notebook/templates/survival.tpl +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/notebook/templates/tabular_timeseries.tpl +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/notebook/templates/vision_from_scratch.tpl +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/raw_mode.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/raw_mode_frame.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/raw_mode_seed.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/raw_mode_survival.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/augmentation.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/callbacks.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/converted.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/features.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/image_size.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/lora.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/lora_validation.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/loss.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/memory_estimate.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/plan.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/preprocessing.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/shape.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/subdataset.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/submitter.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/training/summary.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/lifelines_time_to_event.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/scikit_survival_time_to_event.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/sequence_length.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/skl_tabular_base.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/skl_tabular_classifier.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/skl_tabular_regression.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/statistical_base.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/survival_base.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_causal_language_modeler.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_embeddings.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_image_classifier.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_key_point_detector.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_masked_language_modeler.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_nlp_base.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_object_detector.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_semantic_segmentation.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_sentence_pair_classifier.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_seq2seq_modeler.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_tabular_classifier.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_tabular_regression.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_text_classifier.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_time_series_classifier.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_time_series_forecaster.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_time_to_event.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/upload/torch_token_classifier.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/class_order.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/constants.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/custom_loss_probe.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/detection_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/entry_point.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/feature_dict.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/general_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/key_point_detection_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/registry.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/semantic_segmentation_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/tabular_classification_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/tabular_custom_loss_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/tabular_regression_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/text_classification_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/time_series_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/time_to_event_utils.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/utils/warnings_once.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/validation/__init__.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/validation/checker.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/validation/header_rules.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/validation/lifelines_checks.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/validation/pytorch_checks.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/validation/rewriter.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/validation/scikit_survival_checks.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc/validation/sklearn_checks.py +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc.egg-info/dependency_links.txt +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc.egg-info/top_level.txt +0 -0
- {tracebloc-1.2.71 → tracebloc-1.2.72}/tracebloc_package/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tracebloc
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.72
|
|
4
4
|
Summary: Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace.
|
|
5
5
|
Author-email: tracebloc <lukas@tracebloc.io>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -50,6 +50,29 @@ Provides-Extra: scikit-survival
|
|
|
50
50
|
Requires-Dist: scikit-survival>=0.27.0; extra == "scikit-survival"
|
|
51
51
|
Provides-Extra: all
|
|
52
52
|
Requires-Dist: tracebloc[lifelines,pytorch,scikit-survival,sklearn]; extra == "all"
|
|
53
|
+
Provides-Extra: training
|
|
54
|
+
Requires-Dist: numpy==2.1.3; extra == "training"
|
|
55
|
+
Requires-Dist: pandas==2.3.0; extra == "training"
|
|
56
|
+
Requires-Dist: psutil==7.0.0; extra == "training"
|
|
57
|
+
Requires-Dist: pillow==13.0.0; extra == "training"
|
|
58
|
+
Requires-Dist: safetensors>=0.4.3; extra == "training"
|
|
59
|
+
Requires-Dist: torch==2.14.1; extra == "training"
|
|
60
|
+
Requires-Dist: torchvision==0.29.1; extra == "training"
|
|
61
|
+
Requires-Dist: torchmetrics==1.9.0; extra == "training"
|
|
62
|
+
Requires-Dist: timm==1.0.26; extra == "training"
|
|
63
|
+
Requires-Dist: pycox==0.3.0; extra == "training"
|
|
64
|
+
Requires-Dist: pytorch-forecasting==1.4.0; extra == "training"
|
|
65
|
+
Requires-Dist: transformers==5.16.1; extra == "training"
|
|
66
|
+
Requires-Dist: peft==0.19.1; extra == "training"
|
|
67
|
+
Requires-Dist: torchao==0.17.0; extra == "training"
|
|
68
|
+
Requires-Dist: scikit-learn==1.8.0; extra == "training"
|
|
69
|
+
Requires-Dist: interpret-core==0.6.10; extra == "training"
|
|
70
|
+
Requires-Dist: catboost==1.2.10; extra == "training"
|
|
71
|
+
Requires-Dist: lightgbm==4.6.0; extra == "training"
|
|
72
|
+
Requires-Dist: xgboost-cpu==3.2.0; sys_platform == "linux" and extra == "training"
|
|
73
|
+
Requires-Dist: xgboost==3.2.0; sys_platform != "linux" and extra == "training"
|
|
74
|
+
Requires-Dist: lifelines==0.30.3; extra == "training"
|
|
75
|
+
Requires-Dist: scikit-survival==0.27.0; extra == "training"
|
|
53
76
|
Provides-Extra: dev
|
|
54
77
|
Requires-Dist: pytest==9.2.0; extra == "dev"
|
|
55
78
|
Requires-Dist: pytest-cov==7.1.0; extra == "dev"
|
|
@@ -57,7 +80,7 @@ Requires-Dist: coverage==7.17.0; extra == "dev"
|
|
|
57
80
|
Requires-Dist: jsonschema>=4.18; extra == "dev"
|
|
58
81
|
Requires-Dist: setuptools>=77.0.0; extra == "dev"
|
|
59
82
|
Requires-Dist: mypy>=1.5; extra == "dev"
|
|
60
|
-
Requires-Dist: ruff
|
|
83
|
+
Requires-Dist: ruff==0.16.0; extra == "dev"
|
|
61
84
|
Requires-Dist: sentencepiece>=0.2.0; extra == "dev"
|
|
62
85
|
Requires-Dist: protobuf>=5.29.1; extra == "dev"
|
|
63
86
|
Requires-Dist: einops==0.8.2; extra == "dev"
|
|
@@ -89,8 +112,8 @@ asks for `pip install 'tracebloc[lifelines]'`, a scikit-survival one for
|
|
|
89
112
|
`tracebloc[scikit-survival]`. The extras are the
|
|
90
113
|
`[project.optional-dependencies]` groups in [pyproject.toml](pyproject.toml).
|
|
91
114
|
|
|
92
|
-
Requires Python 3.11–3.14.
|
|
93
|
-
|
|
115
|
+
Requires Python 3.11–3.14. A current macOS or Linux `python3` is inside
|
|
116
|
+
that range; if pip tells you *"No matching distribution found for
|
|
94
117
|
tracebloc"*, check `python3 --version` first — that is what an out-of-range
|
|
95
118
|
interpreter looks like, not a missing package.
|
|
96
119
|
|
|
@@ -115,7 +138,7 @@ training_plan = user.link_model_dataset(dataset_id="<your-dataset-key>")
|
|
|
115
138
|
training_plan.start()
|
|
116
139
|
```
|
|
117
140
|
|
|
118
|
-
For a full walkthrough
|
|
141
|
+
For a full walkthrough in your own notebook, follow [Start training](https://docs.tracebloc.io/join-use-case/start-training) with the [quickstart notebook](https://github.com/tracebloc/quickstart/blob/main/notebooks/traceblocTrainingGuide.ipynb).
|
|
119
142
|
|
|
120
143
|
### Logging in from the environment
|
|
121
144
|
|
|
@@ -378,7 +401,7 @@ Skip 2 and 3 for quiet patch releases with no doc-visible surface.
|
|
|
378
401
|
## Links
|
|
379
402
|
|
|
380
403
|
- [Documentation](https://docs.tracebloc.io/)
|
|
381
|
-
- [Quickstart Notebook](https://
|
|
404
|
+
- [Quickstart Notebook](https://github.com/tracebloc/quickstart/blob/main/notebooks/traceblocTrainingGuide.ipynb)
|
|
382
405
|
- [Website](https://tracebloc.io)
|
|
383
406
|
- [GitHub](https://github.com/tracebloc)
|
|
384
407
|
|
|
@@ -24,8 +24,8 @@ asks for `pip install 'tracebloc[lifelines]'`, a scikit-survival one for
|
|
|
24
24
|
`tracebloc[scikit-survival]`. The extras are the
|
|
25
25
|
`[project.optional-dependencies]` groups in [pyproject.toml](pyproject.toml).
|
|
26
26
|
|
|
27
|
-
Requires Python 3.11–3.14.
|
|
28
|
-
|
|
27
|
+
Requires Python 3.11–3.14. A current macOS or Linux `python3` is inside
|
|
28
|
+
that range; if pip tells you *"No matching distribution found for
|
|
29
29
|
tracebloc"*, check `python3 --version` first — that is what an out-of-range
|
|
30
30
|
interpreter looks like, not a missing package.
|
|
31
31
|
|
|
@@ -50,7 +50,7 @@ training_plan = user.link_model_dataset(dataset_id="<your-dataset-key>")
|
|
|
50
50
|
training_plan.start()
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
For a full walkthrough
|
|
53
|
+
For a full walkthrough in your own notebook, follow [Start training](https://docs.tracebloc.io/join-use-case/start-training) with the [quickstart notebook](https://github.com/tracebloc/quickstart/blob/main/notebooks/traceblocTrainingGuide.ipynb).
|
|
54
54
|
|
|
55
55
|
### Logging in from the environment
|
|
56
56
|
|
|
@@ -313,7 +313,7 @@ Skip 2 and 3 for quiet patch releases with no doc-visible surface.
|
|
|
313
313
|
## Links
|
|
314
314
|
|
|
315
315
|
- [Documentation](https://docs.tracebloc.io/)
|
|
316
|
-
- [Quickstart Notebook](https://
|
|
316
|
+
- [Quickstart Notebook](https://github.com/tracebloc/quickstart/blob/main/notebooks/traceblocTrainingGuide.ipynb)
|
|
317
317
|
- [Website](https://tracebloc.io)
|
|
318
318
|
- [GitHub](https://github.com/tracebloc)
|
|
319
319
|
|
|
@@ -4,19 +4,15 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tracebloc"
|
|
7
|
-
version = "1.2.
|
|
7
|
+
version = "1.2.72"
|
|
8
8
|
description = "Build AI with anyone. On data that can't move. SDK for the tracebloc collaborative AI workspace."
|
|
9
9
|
license = "MIT"
|
|
10
10
|
license-files = ["license.txt"]
|
|
11
11
|
# Upper bound tracks the newest interpreter our extras actually have wheels
|
|
12
|
-
# for, NOT the newest one we happen to test on.
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
# path the product's own "Start training" button sends users down, so a cap
|
|
17
|
-
# below the current Colab runtime breaks the only documented way to launch a
|
|
18
|
-
# training run. Widen deliberately, and only re-narrow with a measured
|
|
19
|
-
# resolution failure to point at.
|
|
12
|
+
# for, NOT the newest one we happen to test on. A cap below the `python3` a
|
|
13
|
+
# user's own notebook runs on makes pip answer "No matching distribution found
|
|
14
|
+
# for tracebloc", which reads like the package does not exist. Widen
|
|
15
|
+
# deliberately, and only re-narrow with a measured resolution failure to point at.
|
|
20
16
|
requires-python = ">=3.11,<3.15"
|
|
21
17
|
readme = "README.md"
|
|
22
18
|
authors = [
|
|
@@ -116,6 +112,37 @@ scikit-survival = [
|
|
|
116
112
|
all = [
|
|
117
113
|
"tracebloc[pytorch,sklearn,lifelines,scikit-survival]",
|
|
118
114
|
]
|
|
115
|
+
# The training images' exact versions of every library above, which the
|
|
116
|
+
# converter's version check holds a conversion to: `pip install
|
|
117
|
+
# "tracebloc[training]"`. Generated from tracebloc/contracts/training_requirements.txt
|
|
118
|
+
# by scripts/sync_training_extra.py; tests/test_sync_training_extra.py fails when
|
|
119
|
+
# one moves without the other.
|
|
120
|
+
# BEGIN training extra: generated by scripts/sync_training_extra.py, do not edit
|
|
121
|
+
training = [
|
|
122
|
+
"numpy==2.1.3",
|
|
123
|
+
"pandas==2.3.0",
|
|
124
|
+
"psutil==7.0.0",
|
|
125
|
+
"pillow==13.0.0",
|
|
126
|
+
"safetensors>=0.4.3",
|
|
127
|
+
"torch==2.14.1",
|
|
128
|
+
"torchvision==0.29.1",
|
|
129
|
+
"torchmetrics==1.9.0",
|
|
130
|
+
"timm==1.0.26",
|
|
131
|
+
"pycox==0.3.0",
|
|
132
|
+
"pytorch-forecasting==1.4.0",
|
|
133
|
+
"transformers==5.16.1",
|
|
134
|
+
"peft==0.19.1",
|
|
135
|
+
"torchao==0.17.0",
|
|
136
|
+
"scikit-learn==1.8.0",
|
|
137
|
+
"interpret-core==0.6.10",
|
|
138
|
+
"catboost==1.2.10",
|
|
139
|
+
"lightgbm==4.6.0",
|
|
140
|
+
"xgboost-cpu==3.2.0; sys_platform == 'linux'",
|
|
141
|
+
"xgboost==3.2.0; sys_platform != 'linux'",
|
|
142
|
+
"lifelines==0.30.3",
|
|
143
|
+
"scikit-survival==0.27.0",
|
|
144
|
+
]
|
|
145
|
+
# END training extra
|
|
119
146
|
dev = [
|
|
120
147
|
# The test tools, at org-config's test-pins.yml versions (its scripts/test_pins.py
|
|
121
148
|
# holds these lines to them); coverage is pytest-cov's, pinned so it cannot float.
|
|
@@ -131,7 +158,7 @@ dev = [
|
|
|
131
158
|
# the same floor as [build-system].
|
|
132
159
|
"setuptools>=77.0.0",
|
|
133
160
|
"mypy>=1.5",
|
|
134
|
-
"ruff
|
|
161
|
+
"ruff==0.16.0",
|
|
135
162
|
# tests/test_convert_tokenizers.py converts a SentencePiece .model and
|
|
136
163
|
# checks its ids against sentencepiece itself; transformers reads the
|
|
137
164
|
# .model through protobuf. Those tests fail, not skip, when CI lacks them.
|
|
@@ -160,8 +187,10 @@ include = ["tracebloc", "tracebloc.*", "tracebloc_package", "tracebloc_package.*
|
|
|
160
187
|
# The backend's stage -> API URL contract (and its pin), which
|
|
161
188
|
# tracebloc.config reads at import: a wheel or sdist without it cannot
|
|
162
189
|
# import the SDK at all. tests/test_stage_urls_contract.py holds this line.
|
|
190
|
+
# The training images' pins, which the converter's version check reads;
|
|
191
|
+
# tests/test_convert_training_versions.py holds the txt line.
|
|
163
192
|
[tool.setuptools.package-data]
|
|
164
|
-
tracebloc = ["contracts/*.json"]
|
|
193
|
+
tracebloc = ["contracts/*.json", "contracts/*.txt"]
|
|
165
194
|
# The per-family notebook templates tracebloc.notebook renders; without them
|
|
166
195
|
# notebook.render cannot render anything. tests/test_notebook_render.py
|
|
167
196
|
# holds this line.
|
|
@@ -31,8 +31,9 @@ _PYPI_EXTRAS_URL = "https://pypi.org/project/tracebloc/"
|
|
|
31
31
|
_DISTRIBUTION = "tracebloc"
|
|
32
32
|
|
|
33
33
|
# Extras that are not "the framework a model needs": ``all`` bundles every
|
|
34
|
-
# framework extra, ``
|
|
35
|
-
|
|
34
|
+
# framework extra, ``training`` pins them all at the training images'
|
|
35
|
+
# versions, ``dev`` is the test/lint toolchain.
|
|
36
|
+
_META_EXTRAS = frozenset({"all", "training", "dev"})
|
|
36
37
|
|
|
37
38
|
# Top-level import names whose distribution name is not the import name
|
|
38
39
|
# with ``_`` → ``-``. This is import-name knowledge only; WHICH extra ships
|
|
@@ -10,14 +10,20 @@ A file the peer wrote (Layer 0) is never rewritten here.
|
|
|
10
10
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
|
+
import os
|
|
13
14
|
import re
|
|
15
|
+
import shutil
|
|
16
|
+
import tempfile
|
|
17
|
+
from contextlib import contextmanager
|
|
14
18
|
from dataclasses import replace
|
|
15
|
-
from
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, Iterator, Mapping, Sequence
|
|
16
21
|
|
|
17
22
|
from tracebloc.convert.gate import ConversionOutcome, _tracebloc_version, convert
|
|
18
23
|
from tracebloc.convert.interfaces import (
|
|
19
24
|
COUNT_NAMES,
|
|
20
25
|
ComparedOutput,
|
|
26
|
+
ConversionRefusedError,
|
|
21
27
|
Provenance,
|
|
22
28
|
SourcedValue,
|
|
23
29
|
UseCaseTask,
|
|
@@ -110,6 +116,51 @@ def changed_counts(
|
|
|
110
116
|
return changed
|
|
111
117
|
|
|
112
118
|
|
|
119
|
+
@contextmanager
|
|
120
|
+
def _restored_on_error(directory: Path) -> Iterator[None]:
|
|
121
|
+
"""Leave ``directory`` as it was if the block raises.
|
|
122
|
+
|
|
123
|
+
The weights, model file and report are written one after another; a failure
|
|
124
|
+
between them must not leave files that disagree about the head. The snapshot
|
|
125
|
+
sits beside ``directory`` so the put-back renames; if it fails, the snapshot
|
|
126
|
+
stays on disk as the only intact copy.
|
|
127
|
+
"""
|
|
128
|
+
scratch = Path(tempfile.mkdtemp(prefix=".tracebloc_saved_", dir=directory.parent))
|
|
129
|
+
saved = scratch / "package"
|
|
130
|
+
try:
|
|
131
|
+
shutil.copytree(directory, saved)
|
|
132
|
+
except BaseException:
|
|
133
|
+
shutil.rmtree(scratch, ignore_errors=True)
|
|
134
|
+
raise
|
|
135
|
+
try:
|
|
136
|
+
yield
|
|
137
|
+
except BaseException:
|
|
138
|
+
_put_back(saved, directory)
|
|
139
|
+
shutil.rmtree(scratch, ignore_errors=True)
|
|
140
|
+
raise
|
|
141
|
+
shutil.rmtree(scratch, ignore_errors=True)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _remove(entry: Path) -> None:
|
|
145
|
+
if entry.is_dir() and not entry.is_symlink():
|
|
146
|
+
shutil.rmtree(entry)
|
|
147
|
+
else:
|
|
148
|
+
entry.unlink()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _put_back(saved: Path, directory: Path) -> None:
|
|
152
|
+
"""Make ``directory`` hold exactly ``saved``'s entries, moving each in whole."""
|
|
153
|
+
kept = {entry.name for entry in saved.iterdir()}
|
|
154
|
+
for entry in directory.iterdir():
|
|
155
|
+
if entry.name not in kept:
|
|
156
|
+
_remove(entry)
|
|
157
|
+
for entry in saved.iterdir():
|
|
158
|
+
target = directory / entry.name
|
|
159
|
+
if target.exists() or target.is_symlink():
|
|
160
|
+
_remove(target)
|
|
161
|
+
os.replace(entry, target)
|
|
162
|
+
|
|
163
|
+
|
|
113
164
|
def _folded(name: Any) -> str:
|
|
114
165
|
return str(name).strip().casefold()
|
|
115
166
|
|
|
@@ -247,12 +298,13 @@ def align_head(
|
|
|
247
298
|
note = f"{head} starts fresh: the checkpoint's classes are not the dataset's"
|
|
248
299
|
compared = ComparedOutput.BACKBONE
|
|
249
300
|
said = "the head started fresh for the dataset's classes"
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
301
|
+
with _restored_on_error(written.directory):
|
|
302
|
+
save_file(state, str(weights))
|
|
303
|
+
task = replace(outcome.task, class_names=order)
|
|
304
|
+
rewritten = _rewrite(
|
|
305
|
+
outcome, header, values, (*package.notes, note), task, compared
|
|
306
|
+
)
|
|
307
|
+
return rewritten, said
|
|
256
308
|
|
|
257
309
|
|
|
258
310
|
def regenerated_for_link(
|
|
@@ -265,8 +317,8 @@ def regenerated_for_link(
|
|
|
265
317
|
"""The package to upload again before the link is retried, and what changed.
|
|
266
318
|
|
|
267
319
|
``conversion`` is the outcome the last upload sent; ``content`` the link
|
|
268
|
-
answer. ``None`` when the upload was not converted,
|
|
269
|
-
fits the dataset.
|
|
320
|
+
answer. ``None`` when the upload was not converted, the package already
|
|
321
|
+
fits the dataset, or the converter refuses the dataset's counts.
|
|
270
322
|
"""
|
|
271
323
|
if conversion is None or not conversion.converted or conversion.package is None:
|
|
272
324
|
return None
|
|
@@ -284,7 +336,16 @@ def regenerated_for_link(
|
|
|
284
336
|
)
|
|
285
337
|
source = conversion.source
|
|
286
338
|
model = source.path if source.path is not None else source.live_object
|
|
287
|
-
|
|
339
|
+
try:
|
|
340
|
+
with _restored_on_error(conversion.written.directory):
|
|
341
|
+
outcome = convert(
|
|
342
|
+
model,
|
|
343
|
+
task=task,
|
|
344
|
+
out_dir=conversion.written.directory,
|
|
345
|
+
licence_confirmed=getattr(conversion.licence, "confirmation", None),
|
|
346
|
+
)
|
|
347
|
+
except ConversionRefusedError:
|
|
348
|
+
return None # the link's mismatch report names the counts
|
|
288
349
|
said.extend(f"{name} {old} -> {new}" for name, (old, new) in changed.items())
|
|
289
350
|
aligned = align_head(outcome, order)
|
|
290
351
|
if aligned is not None:
|
|
@@ -10,6 +10,11 @@ link request names it (``stored_model`` / ``template``) in place of
|
|
|
10
10
|
check read off the file, and answers with the stored name the experiment
|
|
11
11
|
starts with.
|
|
12
12
|
|
|
13
|
+
A library entry (``library_model_id=`` with ``library_version=`` and, to start
|
|
14
|
+
from weights, ``weights_file_id=``) is fetched and checked the same way and
|
|
15
|
+
linked by its ids: the backend runs the upload checks on it again for the
|
|
16
|
+
dataset and answers with the stored name of the copy those checks keep.
|
|
17
|
+
|
|
13
18
|
Pure helpers, no ``User`` state: the user module keeps the orchestration.
|
|
14
19
|
"""
|
|
15
20
|
|
|
@@ -26,6 +31,24 @@ from .utils.general_utils import format_exception
|
|
|
26
31
|
# The link request's field for each kind of stored model.
|
|
27
32
|
STORED_MODEL = "stored_model"
|
|
28
33
|
TEMPLATE = "template"
|
|
34
|
+
LIBRARY_MODEL_ID = "library_model_id"
|
|
35
|
+
LIBRARY_VERSION = "library_version"
|
|
36
|
+
WEIGHTS_FILE_ID = "weights_file_id"
|
|
37
|
+
|
|
38
|
+
#: The ``code`` of each refusal of a library link the backend names, and what
|
|
39
|
+
#: the person does about it. Its ``message`` comes first.
|
|
40
|
+
LIBRARY_REFUSAL_HINTS = {
|
|
41
|
+
"LIBRARY_PICK_NOT_FOUND": (
|
|
42
|
+
"Pick it again from your library: the model, version or weights file "
|
|
43
|
+
"may have been deleted, or its use case no longer allows downloading "
|
|
44
|
+
"its weights."
|
|
45
|
+
),
|
|
46
|
+
"LIBRARY_PICK_CHECK_FAILED": (
|
|
47
|
+
"Upload a fixed version of the model, then link that version."
|
|
48
|
+
),
|
|
49
|
+
"ONE_MODEL_NAME": "Link one model at a time.",
|
|
50
|
+
"MANDATORY_FIELDS": "Pass library_model_id with library_version.",
|
|
51
|
+
}
|
|
29
52
|
|
|
30
53
|
#: The backend's ``LOAD_FAILED`` for a stored model means it holds no live check
|
|
31
54
|
#: of the caller's under that name -- never checked, expired, or someone else's.
|
|
@@ -36,17 +59,61 @@ STORED_MODEL_HINT = (
|
|
|
36
59
|
)
|
|
37
60
|
|
|
38
61
|
|
|
39
|
-
def
|
|
62
|
+
def library_pick(library_model_id, library_version, weights_file_id):
|
|
63
|
+
"""The link reference to a library entry, or ``None`` when none is named.
|
|
64
|
+
A version or weights file without its model, or a model without its
|
|
65
|
+
version, is a :class:`DatasetLinkingError`."""
|
|
66
|
+
if library_model_id is None:
|
|
67
|
+
if library_version is not None or weights_file_id is not None:
|
|
68
|
+
raise DatasetLinkingError(
|
|
69
|
+
"Pass library_model_id= with library_version= and "
|
|
70
|
+
"weights_file_id=."
|
|
71
|
+
)
|
|
72
|
+
return None
|
|
73
|
+
if library_version is None:
|
|
74
|
+
raise DatasetLinkingError(
|
|
75
|
+
"Pass library_version= with library_model_id=: the version's number "
|
|
76
|
+
"your library lists."
|
|
77
|
+
)
|
|
78
|
+
pick = {LIBRARY_MODEL_ID: library_model_id, LIBRARY_VERSION: library_version}
|
|
79
|
+
if weights_file_id is not None:
|
|
80
|
+
pick[WEIGHTS_FILE_ID] = weights_file_id
|
|
81
|
+
return pick
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def raise_library_refusal(response, reference):
|
|
85
|
+
"""Raise the :class:`DatasetLinkingError` for a refused link of the library
|
|
86
|
+
entry ``reference`` names, when the backend names the refusal by its
|
|
87
|
+
``code``; any other answer is left to the general link mapping."""
|
|
88
|
+
if not reference or LIBRARY_MODEL_ID not in reference:
|
|
89
|
+
return
|
|
90
|
+
if response.status_code < 400:
|
|
91
|
+
return
|
|
92
|
+
try:
|
|
93
|
+
body = response.json()
|
|
94
|
+
except ValueError:
|
|
95
|
+
return
|
|
96
|
+
if not isinstance(body, dict) or body.get("code") not in LIBRARY_REFUSAL_HINTS:
|
|
97
|
+
return
|
|
98
|
+
message = body.get("message")
|
|
99
|
+
hint = LIBRARY_REFUSAL_HINTS[body["code"]]
|
|
100
|
+
raise DatasetLinkingError(f"{message}\n{hint}" if message else hint)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def fetch_stored_model(user, dataset_id, model, template, directory, library=None):
|
|
40
104
|
"""Fetch the stored model into ``directory`` (a new temporary one if None).
|
|
41
105
|
|
|
106
|
+
``library`` is :func:`library_pick`'s reference to a library entry.
|
|
42
107
|
Returns ``(stem, paths, reference)``: the path ``upload_model``'s checks
|
|
43
108
|
take, :func:`tracebloc.fetch.fetch_model`'s ``paths``, and the link
|
|
44
|
-
request's reference to the model. ``model
|
|
45
|
-
:class:`DatasetLinkingError`, raised before anything
|
|
109
|
+
request's reference to the model. More than one of ``model``, ``template``
|
|
110
|
+
and ``library`` is a :class:`DatasetLinkingError`, raised before anything
|
|
111
|
+
is fetched.
|
|
46
112
|
"""
|
|
47
|
-
if
|
|
113
|
+
if sum(named is not None for named in (model, template, library)) > 1:
|
|
48
114
|
raise DatasetLinkingError(
|
|
49
|
-
"Pass
|
|
115
|
+
"Pass one of model=, template= or library_model_id= to "
|
|
116
|
+
"link_model_dataset, not more."
|
|
50
117
|
)
|
|
51
118
|
if directory is None:
|
|
52
119
|
directory = tempfile.mkdtemp(prefix="tracebloc-link-")
|
|
@@ -54,6 +121,15 @@ def fetch_stored_model(user, dataset_id, model, template, directory):
|
|
|
54
121
|
# model path), so the directory outlives the call: it goes with the
|
|
55
122
|
# interpreter. A directory the caller named is theirs and stays.
|
|
56
123
|
atexit.register(shutil.rmtree, directory, ignore_errors=True)
|
|
124
|
+
if library is not None:
|
|
125
|
+
paths = user.fetch_library_model(
|
|
126
|
+
library[LIBRARY_MODEL_ID],
|
|
127
|
+
library[LIBRARY_VERSION],
|
|
128
|
+
library.get(WEIGHTS_FILE_ID),
|
|
129
|
+
directory=directory,
|
|
130
|
+
)
|
|
131
|
+
stem = os.path.splitext(paths["model"])[0]
|
|
132
|
+
return _posix_stem(stem), paths, dict(library)
|
|
57
133
|
if template is not None:
|
|
58
134
|
paths = user.fetch_zoo_template(template, dataset_id, directory=directory)
|
|
59
135
|
stem = os.path.splitext(paths["model"])[0]
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""``upload_model``'s front door: any model in, the file today's upload takes out.
|
|
2
|
+
|
|
3
|
+
A ``.py`` or ``.zip`` whose header today's checks accept goes down today's path
|
|
4
|
+
untouched. Anything else (a Hugging Face folder, a checkpoint, the peer's own
|
|
5
|
+
code, a live object) goes through the converter, whose package is uploaded
|
|
6
|
+
only once its acceptance check passed and its licence is confirmed. The use
|
|
7
|
+
case's task comes from ``dataset_id``: ``dataset/<dataset_id>/task/`` answers
|
|
8
|
+
it for a dataset the peer can link a model to.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import contextlib
|
|
14
|
+
import io
|
|
15
|
+
import os
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import Any, Mapping
|
|
18
|
+
from urllib.parse import quote
|
|
19
|
+
|
|
20
|
+
from tracebloc.convert.gate import (
|
|
21
|
+
ConversionOutcome,
|
|
22
|
+
convert,
|
|
23
|
+
resolve_model_file,
|
|
24
|
+
upload_file_for,
|
|
25
|
+
)
|
|
26
|
+
from tracebloc.convert.interfaces import COUNT_NAMES, UseCaseTask
|
|
27
|
+
from tracebloc.convert.licence import ask_to_confirm
|
|
28
|
+
from tracebloc.exceptions import APIError, ModelValidationError
|
|
29
|
+
from tracebloc.utils.constants import TaskCategory
|
|
30
|
+
from tracebloc.utils.general_utils import format_exception
|
|
31
|
+
from tracebloc.validation.header_rules import NO_OUTPUT_CLASSES
|
|
32
|
+
|
|
33
|
+
#: A dataset's category and counts, for a dataset the peer can link a model to.
|
|
34
|
+
DATASET_TASK_PATH = "dataset/{dataset_id}/task/"
|
|
35
|
+
#: The use cases the peer can see, each with its training dataset.
|
|
36
|
+
USE_CASES_PATH = "competition/"
|
|
37
|
+
#: ``licence_confirmed``'s default: ask the peer, in a notebook only.
|
|
38
|
+
ASK = "ask"
|
|
39
|
+
#: How many pages of use cases are read before giving up on a dataset.
|
|
40
|
+
MAX_PAGES = 20
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class UploadInput:
|
|
45
|
+
"""What today's upload is called with, and the conversion behind it."""
|
|
46
|
+
|
|
47
|
+
model_name: Any
|
|
48
|
+
weights: bool
|
|
49
|
+
tokenizer: str | None
|
|
50
|
+
conversion: ConversionOutcome | None = None
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def arguments(self) -> tuple[Any, bool, str | None]:
|
|
54
|
+
return self.model_name, self.weights, self.tokenizer
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _json(response: Any, what: str) -> Any:
|
|
58
|
+
"""``response``'s JSON body, or an ``APIError`` saying ``what`` was unreadable."""
|
|
59
|
+
try:
|
|
60
|
+
return response.json()
|
|
61
|
+
except ValueError as error:
|
|
62
|
+
raise APIError(
|
|
63
|
+
response.status_code,
|
|
64
|
+
f"{what} could not be read (unreadable answer: {format_exception(error)})",
|
|
65
|
+
) from error
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _category(dataset: Mapping[str, Any], dataset_id: str) -> TaskCategory:
|
|
69
|
+
try:
|
|
70
|
+
return TaskCategory(dataset["category"])
|
|
71
|
+
except (KeyError, ValueError) as error:
|
|
72
|
+
raise APIError(
|
|
73
|
+
200,
|
|
74
|
+
f"dataset '{dataset_id}' has no known task category "
|
|
75
|
+
f"({format_exception(error)})",
|
|
76
|
+
) from error
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _not_found(dataset_id: str) -> ModelValidationError:
|
|
80
|
+
return ModelValidationError(
|
|
81
|
+
f"'{dataset_id}' is not the key of a dataset you can link a model to, so "
|
|
82
|
+
"its task is unknown. Check the dataset key on the use case's page."
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _route_missing(response: Any) -> bool:
|
|
87
|
+
"""True for a 404 the server's URL routing answered rather than the view:
|
|
88
|
+
the API's own 404 is JSON ``{"detail": ...}``, an unknown URL's page is not."""
|
|
89
|
+
try:
|
|
90
|
+
body = response.json()
|
|
91
|
+
except ValueError:
|
|
92
|
+
return True
|
|
93
|
+
return not (isinstance(body, Mapping) and "detail" in body)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _class_count(dataset: Mapping[str, Any]) -> int | None:
|
|
97
|
+
meta = dataset.get("dataset_meta")
|
|
98
|
+
counts = meta.get("count") if isinstance(meta, Mapping) else None
|
|
99
|
+
labels = counts.get("label") if isinstance(counts, Mapping) else None
|
|
100
|
+
return len(labels) if isinstance(labels, (Mapping, list)) and labels else None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _use_cases(client: Any) -> Any:
|
|
104
|
+
"""Every use case ``competition/`` lists for the peer, page by page."""
|
|
105
|
+
for page in range(1, MAX_PAGES + 1):
|
|
106
|
+
response = client.get(
|
|
107
|
+
USE_CASES_PATH, params={"page": page}, raise_on_error=False
|
|
108
|
+
)
|
|
109
|
+
if response.status_code != 200:
|
|
110
|
+
raise APIError(response.status_code, "the use cases could not be read")
|
|
111
|
+
body = _json(response, "the use cases")
|
|
112
|
+
yield from body.get("results", []) if isinstance(body, Mapping) else body
|
|
113
|
+
if not isinstance(body, Mapping) or not body.get("next"):
|
|
114
|
+
return
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _task_from_use_cases(client: Any, dataset_id: str) -> UseCaseTask:
|
|
118
|
+
"""The task of the use case whose training dataset is ``dataset_id``, for a
|
|
119
|
+
server without ``dataset/<dataset_id>/task/``: its category, and its class
|
|
120
|
+
count when the dataset records its labels."""
|
|
121
|
+
for use_case in _use_cases(client):
|
|
122
|
+
dataset = use_case.get("dataset") or {}
|
|
123
|
+
if dataset_id not in (use_case.get("dataset_key"), dataset.get("dataset_key")):
|
|
124
|
+
continue
|
|
125
|
+
category = _category(dataset, dataset_id)
|
|
126
|
+
classes = _class_count(dataset)
|
|
127
|
+
counts = {}
|
|
128
|
+
if classes is not None and category not in NO_OUTPUT_CLASSES:
|
|
129
|
+
counts["output_classes"] = classes
|
|
130
|
+
return UseCaseTask(category, from_use_case=True, counts=counts)
|
|
131
|
+
raise _not_found(dataset_id)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def dataset_task(client: Any, dataset_id: str) -> UseCaseTask:
|
|
135
|
+
"""The task of ``dataset_id``, a dataset the peer can link a model to.
|
|
136
|
+
|
|
137
|
+
Its category, and every count the server records for it; a class count
|
|
138
|
+
only for a task with a class per label.
|
|
139
|
+
"""
|
|
140
|
+
path = DATASET_TASK_PATH.format(dataset_id=quote(str(dataset_id), safe=""))
|
|
141
|
+
response = client.get(path, raise_on_error=False)
|
|
142
|
+
if response.status_code == 404:
|
|
143
|
+
if _route_missing(response):
|
|
144
|
+
# TODO(tracebloc/tracebloc-py-package#1059): a server older than
|
|
145
|
+
# dataset/<key>/task/ answers the task from its use cases.
|
|
146
|
+
return _task_from_use_cases(client, dataset_id)
|
|
147
|
+
raise _not_found(dataset_id)
|
|
148
|
+
what = f"the task of dataset '{dataset_id}'"
|
|
149
|
+
if response.status_code != 200:
|
|
150
|
+
raise APIError(response.status_code, f"{what} could not be read")
|
|
151
|
+
body = _json(response, what)
|
|
152
|
+
if not isinstance(body, Mapping):
|
|
153
|
+
raise APIError(200, f"{what} could not be read (unreadable answer: {body!r})")
|
|
154
|
+
category = _category(body, dataset_id)
|
|
155
|
+
counts = {name: body[name] for name in COUNT_NAMES if body.get(name) is not None}
|
|
156
|
+
if category in NO_OUTPUT_CLASSES:
|
|
157
|
+
counts.pop("output_classes", None)
|
|
158
|
+
try:
|
|
159
|
+
return UseCaseTask(category, from_use_case=True, counts=counts)
|
|
160
|
+
except ValueError as error:
|
|
161
|
+
raise APIError(
|
|
162
|
+
200,
|
|
163
|
+
f"{what} could not be read (unreadable answer: {format_exception(error)})",
|
|
164
|
+
) from error
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _todays_path(model: Any) -> bool:
|
|
168
|
+
"""True for what today's upload takes as it is, or refuses with today's message."""
|
|
169
|
+
if not isinstance(model, (str, os.PathLike)):
|
|
170
|
+
return (
|
|
171
|
+
type(model).__module__ == "builtins"
|
|
172
|
+
) # None, a number, a list: not a model
|
|
173
|
+
if not os.fspath(model).strip():
|
|
174
|
+
return True
|
|
175
|
+
return resolve_model_file(model) is None and not os.path.exists(model)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def refuse_unsendable(outcome: ConversionOutcome) -> None:
|
|
179
|
+
"""The upload's refusals of a converted package, before its checks run.
|
|
180
|
+
|
|
181
|
+
The rule is :func:`upload_file_for`'s; its licence line is printed once,
|
|
182
|
+
when the package is sent.
|
|
183
|
+
"""
|
|
184
|
+
with contextlib.redirect_stdout(io.StringIO()):
|
|
185
|
+
upload_file_for(str(outcome.model_file), outcome)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def for_upload(
|
|
189
|
+
model: Any,
|
|
190
|
+
weights: bool,
|
|
191
|
+
tokenizer: str | None,
|
|
192
|
+
*,
|
|
193
|
+
task: UseCaseTask | None,
|
|
194
|
+
licence_confirmed: Any,
|
|
195
|
+
confirmed_by: str | None,
|
|
196
|
+
) -> UploadInput:
|
|
197
|
+
"""What today's upload is called with for ``model``: a path or a live object."""
|
|
198
|
+
if _todays_path(model):
|
|
199
|
+
return UploadInput(model, weights, tokenizer)
|
|
200
|
+
if licence_confirmed == ASK:
|
|
201
|
+
licence_confirmed = ask_to_confirm
|
|
202
|
+
outcome = convert(
|
|
203
|
+
model,
|
|
204
|
+
task=task,
|
|
205
|
+
licence_confirmed=licence_confirmed,
|
|
206
|
+
confirmed_by=confirmed_by,
|
|
207
|
+
)
|
|
208
|
+
if not outcome.converted:
|
|
209
|
+
return UploadInput(model, weights, tokenizer)
|
|
210
|
+
refuse_unsendable(outcome)
|
|
211
|
+
written = outcome.written
|
|
212
|
+
shipped = str(written.tokenizer) if written.tokenizer else tokenizer
|
|
213
|
+
return UploadInput(
|
|
214
|
+
outcome.model_name, written.weights is not None, shipped, outcome
|
|
215
|
+
)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"//": "Pin for tracebloc/contracts/training_requirements.txt, the training images' exact library versions. The txt is tracebloc-engine's use_cases/requirements.txt copied byte-for-byte at `sha`; `blobSha` is its git blob sha there, which tests/test_convert_training_versions.py recomputes. tracebloc.convert.training_versions reads it to refuse a conversion made under other library versions. Re-pin with scripts/sync-training-requirements.sh (put the new commit in `sha` and run it; org-config's contract-pin-bump.yml does the same), then run that test.",
|
|
3
|
+
"repo": "tracebloc/tracebloc-engine",
|
|
4
|
+
"ref": "develop",
|
|
5
|
+
"sha": "e5cc153e6c6ca2bd3eb3d5e42523b7d345e52cde",
|
|
6
|
+
"path": "use_cases/requirements.txt",
|
|
7
|
+
"blobSha": "8e5744a92f2722c2fea81150b263cbf8aa3cf71e",
|
|
8
|
+
"fetched": "2026-10-29"
|
|
9
|
+
}
|